I assume you that to say out of order you need to have this order. That's why I've added identity field into solution. The SQL below will find your S4 transaction.
create table TMSTOCK(id int IDENTITY, trsid varchar(10),
stockin int, stockout int)
insert TMSTOCK VALUES('B1',100,0)
insert TMSTOCK VALUES('S1',0,60)
insert TMSTOCK VALUES('S2',0,30)
insert TMSTOCK VALUES('b2',100,0)
insert TMSTOCK VALUES('s3',0,60)
insert TMSTOCK VALUES('s4',0,60)
insert TMSTOCK VALUES('s5',100,0)
select * from TMSTOCK a
where
(select SUM(stockin)-SUM(stockout) from TMSTOCK b where b.id<=a.id) < 0