Dot matrix by simen · Miscellaneous hardware 500 p...

python
from pathlib import Path
p=Path('/mnt/data/misc_dot-matrix.tar.gz')
p.exists(), p.stat().st_size


(True, 2390332)

python
import tarfile, os, pathlib, json, struct, math, statistics, bisect, re, collections, itertools, numpy as np
out='/mnt/data/dotmatrix_py'
os.makedirs(out,exist_ok=True)
with tarfile.open('/mnt/data/misc_dot-matrix.tar.gz','r:gz') as t:
    print(t.getnames())
    t.extractall(out)
print(list(pathlib.Path(out).rglob('*')))

python
data=Path('/mnt/data/dotmatrix_py/misc_dot-matrix/as1130.logicdata').read_bytes()
class R:
    def __init__(self,b): self.b=b; self.i=0
    def byte(self):
        v=self.b[self.i]; self.i+=1; return v
    def var(self):
        n=self.byte()
        if n==0:return 0
        if n>8: raise ValueError(("bad var",n,self.i-1))
        v=int.from_bytes(self.b[self.i:self.i+n],"little"); self.i+=n; return v
    def skipvar(self,n=1):
        for _ in range(n): self.var()
    def skipid(self): self.skipvar(3)
    def double(self):
        n=self.byte()
        if n>8: raise ValueError(("bad dbl",n,self.i-1))
        raw=self.b[self.i:self.i+n]; self.i+=n; return raw
r=R(data)
magic=r.byte()
first=r.var()
titlelen=r.var()
title=data[r.i:r.i+titlelen].decode(errors='replace'); r.i+=titlelen
block=r.var(); sub=r.var(); freq=r.var(); zero=r.var(); dur=r.var(); freqdiv=r.var(); r.skipvar(2); nch=r.var()
print(magic,first,title,titlelen,block,sub,freq,zero,dur,freqdiv,nch,hex(r.i))
# rest header
r.skipvar();r.skipvar()
r.skipvar()
for _ in range(nch): r.skipid()
r.skipvar()
r.skipvar();r.skipid();r.skipvar();r.skipvar()
print("after header",hex(r.i))
# inline data header
r.skipvar(); r.skipvar(); r.skipvar(5); r.skipvar(); r.skipvar(3); r.skipid(); r.skipvar()
r.skipvar(); r.skipvar(3)
names=[]
for i in range(nch):
    a=r.var(); ch=r.var(); nl=r.var(); name=data[r.i:r.i+nl].decode(errors='replace');r.i+=nl; names.append(name)
    r.skipvar(2); r.double(); r.skipvar(); r.double(); r.skipvar(); r.double()
    if i==nch-1:r.skipvar()
    else:r.skipid();r.skipvar(3)
print(names, hex(r.i))
r.skipvar();r.skipvar(6);r.skipvar(6)
r.skipvar();r.skipvar(2);r.skipvar();r.skipvar();r.skipvar();r.skipvar();r.skipvar();r.skipvar(2);r.skipvar();r.skipvar(2);r.skipvar();r.skipvar(3);r.skipid()
r.skipvar();r.skipvar(3);r.skipvar();r.skipvar();r.skipvar()
r.skipvar();r.skipvar();r.skipvar();r.skipvar(4);r.skipvar()
r.skipvar();r.skipvar();r.skipvar(3);r.skipvar();r.skipvar(3);r.skipid();r.skipvar(6);r.skipvar()
r.skipvar();r.skipvar();r.skipvar();r.skipvar(2);r.skipvar();r.skipvar(3)
print("channel start",hex(r.i), data[r.i:r.i+32].hex())

python
def parse_ch(r,ch):
    start=r.i
    cb=r.var()
    if ch==0: sub=r.var(); blk=r.var()
    cp1=r.var(); z=r.var(); realdur=r.var(); one=r.var(); lastrec=r.var(); left=r.var(); laststate=r.var(); flag=r.var()
    empty=flag==5; long=flag==3
    if ch==0:
        r.skipvar();r.skipvar()
        if empty:
            r.skipvar(18 if long else 11);r.skipvar()
        else:
            while True:
                c=r.byte()
                if c==0: continue
                if c==1:
                    c2=r.byte()
                    if c2==0x18: continue
                    r.i-=2;break
                r.i-=1;break
    else:
        if empty:r.skipvar(10)
        else:
            while True:
                c=r.byte()
                if c!=0:
                    r.i-=1;break
    ne=r.var(); z1=r.var(); ne2=r.var();z2=r.var();ne3=r.var()
    edge_start=r.i
    deltas=[];states=[]
    w=4 if long else 2
    for _ in range(ne):
        raw=int.from_bytes(r.b[r.i:r.i+w],'little');r.i+=w
        if long:
            state=0 if raw&0x80000000 else 1; d=raw&0x7fffffff
        else:
            state=0 if raw&0x8000 else 1; d=raw&0x7fff
        deltas.append(d);states.append(state)
    term=r.byte()
    return dict(start=start,cb=cb,cp1=cp1,realdur=realdur,lastrec=lastrec,left=left,laststate=laststate,flag=flag,
                empty=empty,long=long,ne=ne, ne2=ne2,ne3=ne3,edge_start=edge_start, end=r.i,term=term,
               deltas=np.array(deltas,dtype=np.uint32),states=np.array(states,dtype=np.uint8))
ch0=parse_ch(r,0)
{k:v for k,v in ch0.items() if k not in ('deltas','states')}, len(ch0['deltas']), ch0['deltas'][:10], ch0['states'][:10],hex(r.i)


({'start': 381,
  'cb': 22,
  'cp1': 1,
  'realdur': 1029177344,
  'lastrec': 901861293,
  'left': 127316051,
  'laststate': 1,
  'flag': 2,
  'empty': False,
  'long': False,
  'ne': 1762719,
  'ne2': 1762719,
  'ne3': 26015,
  'edge_start': 441,
  'end': 3525880,
  'term': 0},
 1762719,
 array([767, 242, 262, 243, 261, 244, 262, 243, 262, 243], dtype=uint32),
 array([1, 0, 1, 0, 1, 0, 1, 0, 1, 0], dtype=uint8),
 '0x35ccf8')

python
print(data[420:450].hex())
# Decode vars from around 420 manually
rr=R(data); rr.i=420
for j in range(8):
    pos=rr.i
    try:v=rr.var()
    except Exception as e: print(e);break
    print(hex(pos),v,hex(rr.i))

python
rr=R(data); rr.i=0x1ab
for j in range(8):
    pos=rr.i; v=rr.var(); print(hex(pos),v,hex(rr.i))

python
# Recompute minimal if state valid
pattern=bytes([1,0x16,1,2,0,4])
idx=data.find(pattern,0x35ccf8)
print(hex(idx), data[idx:idx+40].hex())

python
# recreate R with data and position at channel 1 marker
rr=R(data); rr.i=idx
ch1=parse_ch(rr,1)
print({k:v for k,v in ch1.items() if k not in ('deltas','states')})
print(ch1['deltas'][:30])
print(ch1['states'][:30])
print("sum0", int(ch0['deltas'].astype(np.uint64).sum()), "sum1", int(ch1['deltas'].astype(np.uint64).sum()))

python
# Inspect alternation ratio and deltas for shifts around ch1 edge_start
es=ch1['edge_start']
for shift in range(-4,5):
    arr=np.frombuffer(data[es+shift:es+shift+2*4364],dtype='<u2')
    st=(arr<0x8000).astype(np.uint8) # parser state 1 if no sign
    alt=np.mean(st[1:]!=st[:-1])
    med=np.median(arr&0x7fff)
    print(shift, alt, med, arr[:8])

python
vals=ch1['deltas'][:100]
print([(int(v),hex(int(v))) for v in vals[:50]])
print(collections.Counter(map(int,ch1['deltas'])).most_common(30))

python
from PIL import Image
img=Image.open('/mnt/data/dotmatrix_py/misc_dot-matrix/dot-matrix.png').convert('L')
w,h=img.size
arr=np.array(img)
# average top 10% brightness per grid cell
scores=np.zeros((7,17))
for r0 in range(7):
    y0=int(r0*h/7); y1=int((r0+1)*h/7)
    for c0 in range(17):
        x0=int(c0*w/17);x1=int((c0+1)*w/17)
        cell=arr[y0:y1,x0:x1]
        scores[r0,c0]=np.percentile(cell,90)
print(np.round(scores))
print("threshold")
for row in scores:
    print(''.join('#' if v>220 else '.' for v in row))

python
# Rebuild if needed from file quickly using stored? ch0 exists.
for frac in [0,0.1,0.25,0.5,0.75,0.9,0.99]:
    s=int(frac*len(ch0['deltas'])); e=min(s+10000,len(ch0['deltas']))
    d=ch0['deltas'][s:e]; st=ch0['states'][s:e]
    print(frac, float(np.median(d)), float(np.mean(st[1:]!=st[:-1])), int(d.sum()), collections.Counter(map(int,d)).most_common(3))

python
t0=np.cumsum(ch0['deltas'],dtype=np.uint64)
d0=ch0['deltas']
for th in [500,1000,2000,5000,10000,20000,30000]:
    print(th, int(np.sum(d0>th)), float(np.max(d0)))
# segment edges with gaps >1000
idxs=np.where(d0>1000)[0]
print("first gaps", [(int(i),int(d0[i]),int(t0[i])) for i in idxs[:30]])
print("last", [(int(i),int(d0[i]),int(t0[i])) for i in idxs[-20:]])

python
t1=np.cumsum(ch1['deltas'],dtype=np.uint64)
# metadata align global times relative to ch0 raw start
off=(ch1['lastrec']-int(t1[-1]))-(ch0['lastrec']-int(t0[-1]))
print("off",off, "t1first global rel",off+int(t1[0]), "t1last",off+int(t1[-1]), "t0last",int(t0[-1]))
# states of SCL at each t1 trans in t0-relative coords
qt=off+t1.astype(np.int64)
j=np.searchsorted(t0,qt,side='right')
# initial SCL=0; state after j edges: if j=0 0 else ch0 states[j-1]
scl=np.where(j==0,0,ch0['states'][np.clip(j-1,0,len(ch0['states'])-1)])
print("low fraction",np.mean(scl==0),"high",np.mean(scl==1), "range idx",j.min(),j.max())
# distance to nearest SCL edge
prev=np.where(j>0,t0[np.clip(j-1,0,len(t0)-1)],0)
nxt=np.where(j<len(t0),t0[np.clip(j,0,len(t0)-1)],t0[-1])
dist=np.minimum(qt-prev,nxt-qt)
print("dist quantiles",np.quantile(dist,[0,.1,.5,.9,1]))

python
s1=ch1['states']
actual=np.r_[True,s1[1:]!=s1[:-1]]
print(actual.mean(),actual.sum())
print("low fraction actual",np.mean(scl[actual]==0),"repeats",np.mean(scl[~actual]==0))

python
print(ch1['deltas'].min(),ch1['deltas'].max(),np.quantile(ch1['deltas'],[.5,.9,.99,.999,1]))
# repeated-state entries delta distribution vs changes
print("repeat", np.quantile(ch1['deltas'][~actual],[0,.1,.5,.9,.99,1]), len(ch1['deltas'][~actual]))
print("change", np.quantile(ch1['deltas'][actual],[0,.1,.5,.9,.99,1]), len(ch1['deltas'][actual]))

python
es=ch1['edge_start']; n=ch1['ne']
best=[]
for shift in range(-200,1000):
    if es+shift<0 or es+shift+2*n>len(data): continue
    arr=np.frombuffer(data[es+shift:es+shift+2*n],dtype='<u2')
    st=(arr>>15)&1
    alt=np.mean(st[1:]!=st[:-1])
    if alt>0.9:
        best.append((alt,shift,np.median(arr&0x7fff),data[es+shift+2*n]))
sorted(best,reverse=True)[:20]


[]

python
# fresh data/ch1 presumed in state
end1=ch1['end']
print(hex(end1), data[end1:end1+256].hex())
# print hexdump via chunks
for off in range(end1,end1+256,16):
    chunk=data[off:off+16]
    print(f"{off:08x}  "+' '.join(f"{x:02x}" for x in chunk))

python
post1=data[end1:]
knowns={
'last1':ch1['lastrec'].to_bytes(4,'little'),
'last0':ch0['lastrec'].to_bytes(4,'little'),
'dur':ch1['realdur'].to_bytes(4,'little'),
}
for name,bv in knowns.items():
    pos=[]
    st=end1
    while True:
        q=data.find(bv,st)
        if q<0: break
        pos.append(q);st=q+1
    print(name, len(pos), [hex(x) for x in pos[:20]])

python
# reconstruct q t1 transitions and states under metadata alignment
qt=off+t1.astype(np.int64)
# Function SDA state at arbitrary t relative ch0 start. initial state before first qt = opposite first newstate =0
def sample_sda(times):
    inds=np.searchsorted(qt,times,side='right')
    out=np.zeros(len(times),dtype=np.uint8)
    mask=inds>0
    out[mask]=ch1['states'][inds[mask]-1]
    return out
# ch0 rising edge times
rise_mask=ch0['states']==1
trise=t0[rise_mask].astype(np.int64)
# only overlapping qt range
m=(trise>=qt[0])&(trise<=qt[-1])
r=trise[m]
bits=sample_sda(r)
len(r), r[:10], bits[:20]


(23817,
 array([9189400, 9189904, 9190410, 9190915, 9191420, 9191926, 9192431,
        9192937, 9194453, 9194956]),
 array([1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
       dtype=uint8))

python
dr=np.diff(r)
print(collections.Counter(map(int,dr)).most_common(20), np.quantile(dr,[0,.5,.9,.95,.99,1]))
# groups separated by >900
starts=np.r_[0,np.where(dr>900)[0]+1]
ends=np.r_[starts[1:],len(r)]
lens=ends-starts
print(collections.Counter(lens).most_common(20),len(lens), starts[:20], lens[:20])
# convert groups len 9
def byte_from(bits8): return sum(int(b)<<(7-i) for i,b in enumerate(bits8))
vals=[]
for s,e in zip(starts,ends):
    bb=bits[s:e]
    if len(bb)==9: vals.append((byte_from(bb[:8]),int(bb[8]),int(r[s])))
print(len(vals), vals[:100])
print(collections.Counter(v for v,a,t in vals).most_common(20))