# Copyright (c) 2018-2021, Carnegie Mellon University
# See LICENSE for details

Declare(TTensorI_OL);
Declare(TTensorI_OL_Vectorize_AVecLast);

#Class(RewriteBarrier,RewritableObject);
#Class(RewriteBarrier,DFT);
#RewriteBarrier.numops:=self>>1;

DropVectorTag:=function(nt)
    return nt.withoutTag(AVecReg);
#D    return SetTag(nt,Filtered(GetTags(nt),t->ObjId(t)<>AVecReg));
end;

DropParTag:=function(nt)
   return nt.withoutTag(AParSMP);
#D    return SetTag(nt,Filtered(GetTags(nt),t->ObjId(t)<>AParSMP));
end;

Devectorize:=function(l,vlen)
  return List(l,e->
      Cond(
          ObjId(e)=TArray and e.t=TUnknown, TArray(e.t,e.size*vlen),
          ObjId(e)=TArray and ObjId(e.t)=TVect, TArray(e.t.t,e.size*e.t.size),
          ObjId(e)=TVect, TArray(e.t,e.size),
          e)
  );
end;

Class(VTensor_OL, Tensor, rec(
    new := (self, L) >> SPL(WithBases(self, rec(
        _children := [L[1]],
        vlen := L[2]))),
    print := (self,i,is) >> Print(self.name, "(",
        self.child(1).print(i+is,is), ", ", self.vlen,")"),
    sums := self >> Inherit(self, rec(_children := [self.child(1).sums()])),
    isPermutation := False,
    rng := meth(self)         #FULL HACK
       return Devectorize(self.child(1).rng(),self.vlen);
    end,
    dmn := meth(self)         #FULL HACK
       return Devectorize(self.child(1).dmn(),self.vlen);
    end,
    dims := meth(self)
       if (IsBound(self.rng)and IsBound(self.dmn)) then
          return [StripList(List(self.rng(),l->l.size)),
                  StripList(List(self.dmn(),l->l.size))];
       fi;
    end,
));

#VTensor.dmn:=  meth(self)         #FULL HACK
#  local x;
#  x:=self.child(1).dmn()[1];
#  if ObjId(x)=TArray and x.t=TUnknown then
#      return [TArray(TUnknown,x.size*self.vlen)];
#  else
#      return Devectorize(self.child(1).dmn());
#  fi;
#end;
#VTensor.rng:=  meth(self)         #FULL HACK
#  local x;
#  x:=self.child(1).rng()[1];
#  if ObjId(x)=TArray and x.t=TUnknown then
#      return [TArray(TUnknown,x.size*self.vlen)];
#  else
#      return Devectorize(self.child(1).rng());
#  fi;
#end;

BlockVPerm.dmn := meth(self)
       return [TArray(self.child(1).dmn()[1].t,self.child(1).dmn()[1].size*self.n)];
end;
BlockVPerm.rng := meth(self)
       return [TArray(self.child(1).rng()[1].t,self.child(1).rng()[1].size*self.n)];
end;

Class(ScatQuestionMark, Scat, rec());
Class(ICScatAcc,Scat,rec(codeletName:="ICSA"));

Class(VScatQuestionMark, VScat, rec());

Class(VScat_svQuestionMark, VScat_sv, rec());

Class(ScatInit, BaseMat, rec(
   new := meth(self, f,con,c)
        local res;
        res := SPL(WithBases(self, rec(func:=f,cond:=con, _children:=c)));
        return res;
   end,
   rng:=self>>self._children.rng(),
   dmn:=self>>self._children.dmn(),
   print := meth(self,i,is)
      Print(self.name, "(", self.func, ", ",self.cond,", ");
      Print("\n", Blanks(i+is));
      SPLOps.Print(self._children, i + is, is);
      Print("\n", Blanks(i),")");
      return;
   end,
   rChildren := self >> [self._children,self.func],
   rSetChild := meth ( self, n, newChild )
     if n= 1  then
         self._children :=newChild;
     elif n=2 then
         self.func := newChild;
else
        Error("<n> must be 1");
     fi;
   end

));

Class(ScatInitProbe,ScatInit,rec());
Class(ScatInitFixed,ScatInit,rec());

Class(KroneckerSymbol, BaseMat, rec(
   abbrevs := [ arg -> [Flat(arg)] ],
   new := meth(self, l)
        local res;
        res := SPL(WithBases(self, rec(element:=l)));
        return res;
   end,
   isExp:=true,
   dims:=self>>[1,1]  #avoid recursive definition for the old-school Tensor
));

## Base Vector
Class(BV, BaseMat, rec(
   new := (self, i) >> #Checked(IsVar(i),SPL(WithBases(self, rec(element:=i))))
       SPL(WithBases(self, rec(element:=i))),
   dims:=self>>self.element.dimensions #[0,0]  #avoid recursive definition for the old-school Tensor
));

## Multiplication operator
## Multiplication(1,n) is I(n)
## Multiplication(2,n) is a point-wise multiplication of two vectors of size n
 Class(Multiplication, BaseMat, rec(
    abbrevs := [ arg -> [Flat(arg)] ],
    new := meth(self, l)
        local res;
        if l[1]=1 then return Prm(fId(l[2])); fi;
        res := SPL(WithBases(self, rec(element:=l,TType:=Replicate(l[1],TUnknown))));
        return res;
    end,
    isPermutation := self >> false,
#    dmn:=self>>Replicate(self.element[1],TArray(self.TType,self.element[2])),
#HACK
#    dmn:=meth(self) local a; a:=Replicate(self.element[1],TArray(self.TType,self.element[2]));a[1]:=TArray(TReal,1);return a; end,
    dmn:=self >>List(self.TType,x->TArray(x,self.element[2])),
    rng:=self>>let(a:=Try(First(self.TType,x->ObjId(x)=TVect)),t:=Cond(a[1],a[2],self.TType[1]),[TArray(t,self.element[2])]),
    sums:= self>>self,
    numops:=self>>self.element[2]*(self.element[1]-1),
    transpose := self >>self,
    print := meth(self,i,is)
      Print(self.name, "(", self.element[1], ", ",self.element[2],")"); self.printA();
      return;
     end
 ));
 Class(ICMultiplication,Multiplication, rec(

 ));

  _mk_advdim_r := (e) -> List(e, x -> When(IsList(x), _mk_advdim_r(x), [x]));
  _mk_advdim := (d) -> _mk_advdim_r(When(IsList(d), d, [d]));

  Class(Glue, BaseMat, rec(
    abbrevs := [(n,size) ->[n,size]],
    new := meth(self,n,size)
        local res;
        res := SPL(WithBases(self, rec(element:=[n,size],dimensions:=[size*n,Replicate(n,size)],TType:=Replicate(n,TUnknown))));
        return res;
    end,
    isPermutation := self >> false,
    dmn:=self >>List([1..(self.element[1])],x->TArray(self.TType[x],self.element[2])),
    rng:=self>>[TArray(self.TType[1],self.element[1]*self.element[2])],
    sums:= self>>self,
    transpose:=self>>Copy(self),
    numops:=self>>self.element[2]*(self.element[1]-1),
    print := meth(self,i,is)
      Print(self.name, "(", self.element[1], ", ",self.element[2],")"); self.printA();
      return;
     end,
    rChildren := self >> self.element,
    rSetChild := meth( self, n, newChild ) self.element[n] := Checked(n=1 or n=2, newChild); end,
    advdims := self >> let( d := self.dims(), [ _mk_advdim(d[1]), _mk_advdim(d[2]) ]),
    normalizedArithCost := (self) >> 0,
    isReal := self >> true, # makes no sense
 ));

  Class(Split, BaseMat, rec(
    abbrevs := [(size,n) ->[size,n]],
    new := meth(self,size,n)
        local res;
        res := SPL(WithBases(self, rec(element:=[size,n],dimensions:=[Replicate(n,size / n),size],TType:=[TUnknown])));
        return res;
    end,
    isPermutation := self >> false,
    rng:=self >>List([1..self.element[2]],x->TArray(self.TType[1],self.element[1] / self.element[2])),
    dmn:=self>>[TArray(self.TType[1],self.element[1])],
    sums:= self>>self,
    transpose:= self>>Copy(self),
    numops:=self>>(self.element[1]),
    print := meth(self,i,is)
      Print(self.name, "(", self.element[1], ", ",self.element[2],")"); self.printA();
      return;
     end,
    rChildren := self >> self.element,
    rSetChild := meth( self, n, newChild ) self.element[n] := Checked(n=1 or n=2, newChild); end,
    advdims := self >> let( d := self.dims(), [ _mk_advdim(d[1]), _mk_advdim(d[2]) ]),
    normalizedArithCost := (self) >> 0,
    isReal := self >> true, # makes no sense
 ));

## NoOp, used with in COND_OL()
Class(NoOp, BaseMat, rec(
    abbrevs := [ arg -> [Flat(arg)] ],
    new := meth(self, l)
       local res;
        if (Length(l) = 1) then
          return SPL(WithBases(self, rec(element:=l, TType:=Replicate(1, TUnknown))));
        fi;
    end,
    isPermutation := self >> false,
    dmn:=self >> [TArray(self.TType[1], self.element[1])],
    rng:=self >> [TArray(self.TType[1], self.element[1])],
    sums:= self>>self,
    numops:=self>> 0,
    print := meth(self,i,is)
      Print(self.name, "(", self.element[1], ")"); self.printA();
      return;
    end
));

## Constant
Class(Const, BaseMat, rec(
    abbrevs := [ arg -> [Flat(arg)] ],
    new := meth(self, l)
       local res;
        if (Length(l) = 1) then
          return SPL(WithBases(self, rec(element:=l, TType:=Replicate(1, TUnknown))));
        fi;
	# Here
    end,
    isPermutation := self >> false,
    dmn:=self >> [],
# [TArray(self.TType[1], 0)],
    rng:=self >> [TArray(self.TType[1], 1)],
    sums:= self>>self,
    numops:=self>> 0,
    print := meth(self,i,is)
      Print(self.name, "(", self.element[1], ")"); self.printA();
      return;
    end
));

Class(Or, BaseMat, rec(
    abbrevs := [ arg -> [Flat(arg)] ],
    new := meth(self, l)
       local res;
        if (Length(l) = 1) then
          return SPL(WithBases(self, rec(element:=l, TType:=Replicate(l[1], TUnknown))));
        fi;
	# Here
    end,
    isPermutation := self >> false,
    dmn:=self >> List(self.TType, x->TArray(x,1)),
    rng:=self >> [ TArray(self.TType[1], 1)],
    sums:= self>>self,
    numops:=self>> 0,
    print := meth(self,i,is)
      Print(self.name, "(", self.element[1], ")"); self.printA();
      return;
    end
));

Class(And, BaseMat, rec(
    abbrevs := [ arg -> [Flat(arg)] ],
    new := meth(self, l)
       local res;
        if (Length(l) = 1) then
          return SPL(WithBases(self, rec(element:=l, TType:=Replicate(l[1], TUnknown))));
        fi;
	# Here
    end,
    isPermutation := self >> false,
    dmn:=self >> List(self.TType, x->TArray(x,1)),
    rng:=self >> [ TArray(self.TType[1], 1)],
    sums:= self>>self,
    numops:=self>> 0,
    print := meth(self,i,is)
      Print(self.name, "(", self.element[1], ")"); self.printA();
      return;
    end
));

Class(NotEqual, BaseMat, rec(
    abbrevs := [ arg -> [Flat(arg)] ],
    new := meth(self, l)
       local res;
        if (Length(l) = 1) then
          return SPL(WithBases(self, rec(element:=l, TType:=Replicate(l[1], TUnknown))));
        fi;
	# Here
    end,
    isPermutation := self >> false,
    dmn:=self >> List(self.TType, x->TArray(x,1)),
    rng:=self >> [ TArray(self.TType[1], 1)],
    sums:= self>>self,
    numops:=self>> 0,
    print := meth(self,i,is)
      Print(self.name, "(", self.element[1], ")"); self.printA();
      return;
    end
));

Class(Equal, BaseMat, rec(
    abbrevs := [ arg -> [Flat(arg)] ],
    new := meth(self, l)
       local res;
        if (Length(l) = 1) then
          return SPL(WithBases(self, rec(element:=l, TType:=Replicate(l[1], TUnknown))));
        fi;
	# Here
    end,
    isPermutation := self >> false,
    dmn:=self >> List(self.TType, x->TArray(x,1)),
    rng:=self >> [ TArray(self.TType[1], 1)],
    sums:= self>>self,
    numops:=self>> 0,
    print := meth(self,i,is)
      Print(self.name, "(", self.element[1], ")"); self.printA();
      return;
    end
));

Class(ExclusiveOr, BaseMat, rec(
    abbrevs := [ arg -> [Flat(arg)] ],
    new := meth(self, l)
       local res;
        if (Length(l) = 1) then
          return SPL(WithBases(self, rec(element:=l, TType:=Replicate(l[1], TUnknown))));
        fi;
	# Here
    end,
    isPermutation := self >> false,
    dmn:=self >> List(self.TType, x->TArray(x,1)),
    rng:=self >> [ TArray(self.TType[1], 1)],
    sums:= self>>self,
    numops:=self>> 0,
    print := meth(self,i,is)
      Print(self.name, "(", self.element[1], ")"); self.printA();
      return;
    end
));

## Minimums
## Minimums(n) -> Input is n-way vector
Class(Minimums, BaseMat, rec(
    abbrevs := [ arg -> [Flat(arg)] ],
    new := meth(self, l)
       local res;
        if (Length(l) = 1) then
          return SPL(WithBases(self, rec(element:=l, TType:=Replicate(l[1], TUnknown))));
        fi;
	# Here
    end,
    isPermutation := self >> false,
    dmn:=self >> List(self.TType, x->TArray(x,1)),
    rng:=self >> [ TArray(self.TType[1], 1)],
    sums:= self>>self,
    numops:=self>> 0,
    print := meth(self,i,is)
      Print(self.name, "(", self.element[1], ")"); self.printA();
      return;
    end
));

## Maximums
## Maximums(n) -> Input is n-way vector
Class(Maximums, BaseMat, rec(
    abbrevs := [ arg -> [Flat(arg)] ],
    new := meth(self, l)
       local res;
        if (Length(l) = 1) then
          return SPL(WithBases(self, rec(element:=l, TType:=Replicate(l[1], TUnknown))));
        fi;
	# Here
    end,
    isPermutation := self >> false,
    dmn:=self >> List(self.TType, x->TArray(x,1)),
    rng:=self >> [ TArray(self.TType[1], 1) ],
    sums:= self>>self,
    numops:=self>> 0,
    print := meth(self,i,is)
      Print(self.name, "(", self.element[1], ")"); self.printA();
      return;
    end
));

#Addition
Class(Addition, BaseMat, rec(
    abbrevs := [ arg -> [Flat(arg)] ],
    new := meth(self, l)
        local res;
        if l[1]=1 then return Prm(fId(l[2])); fi;
        res := SPL(WithBases(self, rec(element:=l,TType:=Replicate(l[1],TUnknown))));
        return res.setDims();
    end,
    isPermutation := self >> false,

    dmn:=self >> List( [1..self.element[1]], i -> TArray(TUnknown, self.element[2])),
    rng:=self >> [TArray(TUnknown, self.element[2])],
    sums:= self>>self,
    numops:=self>>self.element[2]*(self.element[1]-1),
    print := meth(self,i,is)
      Print(self.name, "(", self.element[1], ", ",self.element[2],")"); self.printA();
      return;
     end,
    transpose := self >> InertTranspose(self),
    area := self >> self.element[2]*(self.element[1]-1),
));

Class(Subtraction, BaseMat, rec(
    abbrevs := [ arg -> [Flat(arg)] ],
    new := meth(self, l)
        local res;
        if l[1]=1 then return Prm(fId(l[2])); fi;
        res := SPL(WithBases(self, rec(element:=l,TType:=Replicate(l[1],TUnknown))));
        return res;
    end,
    isPermutation := self >> false,
    dmn:=self >>List(self.TType,x->TArray(x,self.element[2])),
    rng:=self>>let(a:=Try(First(self.TType,x->ObjId(x)=TVect)),t:=Cond(a[1],a[2],self.TType[1]),[TArray(t,self.element[2])]),
    sums:= self>>self,
    numops:=self>>self.element[2]*(self.element[1]-1),
    print := meth(self,i,is)
      Print(self.name, "(", self.element[1], ", ",self.element[2],")"); self.printA();
      return;
     end
));

# Declare(Cross);
# Class(Cross, BaseOperation, rec(
#    abbrevs := [ arg -> [Flat(arg)] ],

#    new := meth(self, L)
#         if Length(L)=1 then return L[1]; fi;
#         return SPL(WithBases(self, rec(_children:=L)));
#    end,

#    codeletName:="C",
#    isPermutation := self >> false,
#    dmn:=self>>let(li:=[],Flat(List(self._children, x-> x.dmn()))),
#    rng:=self>>let(li:=[],Flat(List(self._children, x-> x.rng()))),
#    sums:= meth(self)
#    local i;
#    for i in [1..Length(self._children)] do
#       self._children[i]:=self._children[i].sums();
#    od;
#    return self;
#    end,
#    hasLeftGath:=meth(self)
#       local i;
#       for i in self._children do
# #          if (ObjId(i)=Compose and ObjId(i.rChildren()[1])=Gath) or
# #              (ObjId(i)=Gath) then
# #              return true;
#            if (ObjId(i)=Compose and IsBound(i.rChildren()[1].func) and i.rChildren()[1].func.free()<>Set([])) or (IsBound(i.func) and (not ObjId(i)=VGath_dup) and i.func.free()<>Set([])) and not(ObjId(i)=ISum) then
#                return true;
#           fi;
#       od;
#       return false;
#    end,
#    hasRightScat:=meth(self)
#       local i;
#       for i in self._children do
# #          if (ObjId(i)=Compose and ObjId(Last(i.rChildren()))=Scat) or
# #              (ObjId(i)=Scat) then
# #              return true;
# #          fi;
#            if (ObjId(i)=Compose and ((IsBound(Last(i.rChildren()).func) and Last(i.rChildren()).func.free()=Set([])) or (ObjId(Last(i.rChildren()))=ISum) or (ObjId(Last(i.rChildren()))=VGath_dup and Length(Last(i.rChildren()).func.free())=1))) or (IsBound(i.func) and i.func.free()=Set([])) or (ObjId(i)=VGath_dup and Length(i.func.free())=1) or ObjId(i)=ISum  then
#                return true;
#           fi;
#       od;
#       return false;
#    end,
#    splitCross:=meth(self)
#       local l,r;
#       l:=Copy(self);
#       r:=[];
#       for i in [1..Length(l._children)] do
# #          if (ObjId(l._children[i])=Compose and ObjId(Last(l._children[i].rChildren()))=Scat) then
#            if (ObjId(l._children[i])=Compose and ((IsBound(Last(l._children[i].rChildren()).func) and Last(l._children[i].rChildren()).func.free()=Set([])) or (ObjId(Last(l._children[i].rChildren()))=ISum) or (ObjId(Last(l._children[i].rChildren()))=VGath_dup and Length(Last(l._children[i].rChildren()).func.free())=1))) or (ObjId(l._children[i])=ISum) then
#               Add(r,Last(l._children[i].rChildren()));
#               l._children[i]._children:=DropLast(l._children[i]._children,1);
# #          elif (ObjId(l._children[i])=Scat) then
#            elif (IsBound(l._children[i].func) and l._children[i].func.free()=Set([])) or (ObjId(i)=VGath_dup and Length(i.func.free())=1) then
#               Add(r,l._children[i]);
#               l._children[i]:=Prm(fId(l._children[i].dims()[1]));
#           else
#               Add(r,Prm(fId(l._children[i].dims()[2])));
#           fi;
#       od;
#       return [l,Cross(r)];
#    end
#  )
# );

Class(CrossBlockTop,Cross);

Identities:=function(l)
   return Cross(List(l,x->Prm(fId(x.size))));
end;

BB.numops:= self>>self.child(1).numops();
NoPull.numops:= self>>self.child(1).numops();
NoPullRight.numops:= self>>self.child(1).numops();
NoPullLeft.numops:= self>>self.child(1).numops();

PushR.numops:= self>>self.child(1).numops();

ISum.numops:=self>>self.child(1).numops()*self.domain;

SUM.numops:=self>>Sum(List(self._children,c->c.numops()));

Cross.numops:=self>>Sum(List(self._children,c->c.numops()));

Compose.numops:=self>>Sum(List(self._children,c->c.numops()));

Scat.numops:=self>>self.dims()[2];

Gath.numops:=self>>self.dims()[1];

VGath.numops:=self>>self.dims()[1];
VGath_dup.numops:=self>>self.dims()[1];
VReplicate.numops:=self>>self.v;
VHAdd.numops:=self>>self.v*self.v;
VPerm.numops:=self>>self.dimensions[1];
Tensor.numops:=self>>self.dimensions[1];

ScatAcc.numops:=self>>self.dimensions[2]*2; #an add and a store
VScatAcc.numops:=self>>self.dimensions[2]*2; #an add and a store
VScat.numops:=self>>self.dimensions[2];

fBase.numops:=self>>0;
fTensor.numops:=self>>0;
fId.numops:=self>>0;
ScatInit.numops:=self>>self._children.numops();

Prm.numops:=self>>0;
I.numops:=self>>0;

VTensor_OL.numops:=self>>self._children[1].numops()*self.vlen;

Class(AOne,AGenericTag);
Class(AMul,AGenericTag, rec(
    __call__ := meth ( arg )
      local  result, self, params;
      self := arg[1];
      params := arg{[ 2 .. Length(arg) ]};
      result := WithBases(self, rec(
              params := params,
              operations := PrintOps));
      return result;
    end,
    print := self >> Print(self.__name__, "(", PrintCS(self.params), ")")
   ));

Class(VOLWrap, VWrapBase, rec(
    __call__ := (self,isa) >> Checked(IsSIMD_ISA(isa),
        WithBases(self, rec(operations:=PrintOps, isa:=isa))),

    wrap := (self,r,t) >> let(isa := self.isa, v := isa.v,
#This is OBVIOUSLY a hack. only deals with some kind of vectorization
            nontransforms.ol.TTensorI_OL_Vectorize_AVecLast(TTensorI_OL(t, [ AOne, AVec ],[ [ 0, 2 ] ], [ 1, v ], [ AVecReg(isa) ]), r)),

    twrap := (self, t) >> let(isa := self.isa, v := isa.v,
#This is OBVIOUSLY a hack. only deals with some kind of vectorization
           TTensorI_OL(t, [ AOne, AVec ],[ [ 0, 2 ] ], [ 1, v ], [ AVecReg(isa) ])
        ),

    print := self >> Print(self.name, "(", self.isa, ")")
));

Class(TTensorI_OL, Tagged_tSPL, rec(
    abbrevs := [
        (nt,g,s,v)  -> [nt,List([1..Length(g)],i->When(v[i]=1,AOne,g[i])),s,v]
    ],

    dmn := self >> let(
        nt := self.params[1],
        sizes := self.params[4],
        List([1..Length(sizes)], i->
            TArray(nt.dmn()[i].t,sizes[i] * nt.dmn()[i].size)
        )),

    rng := self >> let(
        nt := self.params[1],
        s := self.params[3],
        v := self.params[4],
        List([1..Length(s)], i->
            TArray(nt.rng()[i].t, Product(List(s[i], a ->
                When(a=0,
                    nt.rng()[i].size,
                    v[a]
                )
            )))
        )
    ),

    isReal := self >> self.params[1].isReal(),
    doNotMeasure:=true,
    doNotSaveInHashtable:=true,
    decomposePerformance:=true,
    transpose := self >> Copy(self),
#D    tagpos :=5,
));

#Changed by Marek, looks OK
NewRulesFor(TTensorI_OL, rec(
    TTensorI_OL_Base :=rec(
        applicable := (self,t) >> not(t.hasTag(AParSMP)) and t.getTags() = t.params[1].getTags(),
        freedoms := nt -> let(
            g := nt.params[2],
            nbvars := Length(Filtered(g,x->x=APar or x=AVec)),
            [Arrangements([1..nbvars],nbvars)]
        ),
        child := (nt,freedoms) -> [nt.params[1],InfoNt(freedoms)],
        recompose := (nt,cnt,cperf) -> cperf[1]*Product(List(Filtered(Zip2(nt.params[2],nt.params[4]),l->l[1] in [APar,AVec]),x->x[2])),
        apply := function(nt,c,cnt)
            local g,s,v,perm,ind1,ind,ind2,gathers,scatters,result,z;
            g := nt.params[2];
            s := nt.params[3];
            v := nt.params[4];
            perm := cnt[2].params[1][1];
            ind1 := List([1..Length(g)], i ->
                Cond(g[i]=APar or g[i]=AVec, fBase(Ind(v[i])),
                    g[i]=AOne, fId(1),
                    ObjId(g[i])=AMul, g[i],
                    Error("PV not known!")
                )
            );
            ind := List(ind1, x -> When(ObjId(x)=AMul,ind1[x.params[1]],x));
            gathers := Cross(List([1..Length(g)], i -> let(
                kernelsize := fId(cnt[1].dmn()[i].size),
                When(g[i]=APar or (ObjId(g[i])=AMul and g[i].params[2]=APar),
                    Gath(fTensor(ind[i],kernelsize)),
                    Gath(fTensor(kernelsize,ind[i]))
                )
            )));
            scatters := Cross(List([1..Length(s)], i ->
                ScatQuestionMark( fTensor(
                    List(s[i], y ->
                        When(y=0,
                            fId(cnt[1].rng()[i].size),
                            ind[y]
                        )
                    )
                ))
            ));
            result := scatters*c[1]*gathers;
            ind2:=[];
            for i in [1..Length(g)] do
                if g[i]=APar or g[i]=AVec then
                    Add(ind2,ind[i]);
                fi;
            od;
            for i in [1..Length(ind2)] do
                result:=ISum(ind2[perm[i]].params[2], ind2[perm[i]].params[1], result);
            od;

            return result;
        end
    ),
#D        applicable :=(self,t) >> Length(Filtered(GetTags(t),x->ObjId(x)=AParSMP))=0 and GetTags(t)=GetTags(t.params[1])
#D,
#D            freedoms := nt -> let(g:=nt.params[2],
#D                nbvars:=Length(Filtered(g,x->x=APar or x=AVec)),
#D                [Arrangements([1..nbvars],nbvars)]),
#D            child := (nt,freedoms) -> [nt.params[1],InfoNt(freedoms)],
#D            recompose := (nt,cnt,cperf) -> cperf[1]*Product(List(Filtered(Zip2(nt.params[2],nt.params[4]),l->l[1] in [APar,AVec]),x->x[2])),
#D            apply := function(nt,c,cnt)
#D                local g,s,v,perm,ind1,ind,ind2,gathers,scatters,result,z;
#D                g:=nt.params[2];
#D                s:=nt.params[3];
#D                v:=nt.params[4];
#D                perm:=cnt[2].params[1][1];
#D                ind1:=List([1..Length(g)],
#D                    i->Cond(g[i]=APar or g[i]=AVec,fBase(Ind(v[i])),
#D                    g[i]=AOne,fId(1),
#D                    ObjId(g[i])=AMul,g[i],
#D                    Error("PV not known!")));
#D                ind:=List(ind1,x->When(ObjId(x)=AMul,ind1[x.params[1]],x));
#D                gathers:= Cross(List([1..Length(g)],
#D                        i->let(kernelsize:=fId(cnt[1].dmn()[i].size),
#D                            When(g[i]=APar or (ObjId(g[i])=AMul and g[i].params[2]=APar),
#D                                Gath(fTensor(ind[i],kernelsize)),
#D                                Gath(fTensor(kernelsize,ind[i]))))));
#D                scatters:=Cross(List([1..Length(s)],
#D                            i->ScatQuestionMark(fTensor(
#D                                    List(s[i],y->
#D                                        When(y=0,fId(cnt[1].rng()[i].size),
#D                                            ind[y]))))));
#D                result:=scatters*c[1]*gathers;
#D                ind2:=[];
#D                for i in [1..Length(g)] do
#D                    if g[i]=APar or g[i]=AVec then
#D                        Add(ind2,ind[i]);
#D                    fi;
#D                od;
#D                for i in [1..Length(ind2)] do
#D                        result:=ISum(ind2[perm[i]].params[2],ind2[perm[i]].params[1],result);
#D                od;
#D
#D                return result;
#D            end
#D            ),

#Changed by Marek, looks OK
    TTensorI_OL_Parrallelize_AParFirst := rec(
        applicable :=(self,t) >>
            t.isTag(1, AParSMP)
            and 0 <> t.params[3][1][1]
            and APar = t.params[2][t.params[3][1][1]]
            and t.firstTag().params[1] = t.params[4][t.params[3][1][1]]
            and 1 = t.params[3][1][1], #t.params[3][1][1]=1 is a trick to prevent // the 2nd input because TensorGeneral breaks at the moment

        freedoms := nt -> [[1]],

        child := function (nt,freedoms)
            local PV,sizes;
            sizes := Copy(nt.params[4]);
            sizes[nt.params[3][1][1]] := 1;
            if Length(Filtered(sizes, t -> t<>1)) > 0 then
                PV:=Copy(nt.params[2]);
                PV[nt.params[3][1][1]] := AOne;
                return [TTensorI_OL(
                    DropParTag(nt.params[1]),
                    PV,
                    nt.params[3],
                    sizes,
                    Drop(nt.params[5],1)
                )];
            else
                return [ DropParTag(Copy(nt.params[1])) ];
            fi;
        end,

        apply := function(nt,c,cnt)
            local myCross,a,b,index;

            index := Ind(nt.firstTag().params[1]);

            a := List(c[1].dims()[2], x -> fId(x));
            a[nt.params[3][1][1]] := fTensor(fBase(index), a[nt.params[3][1][1]]);

            b:=List(a, x -> Gath(x));

            myCross:=Cross(b);

#              return SMPSum(GetFirstTag(nt).params[1],index, GetFirstTag(nt).params[1],ScatQuestionMark(fTensor(fBase(index), fId(c[1].dims()[1]))) *c[1]* Cross(Gath(fTensor(fBase(index), fId(c[1].dims()[2][1]))),Gath(fId(c[1].dims()[2][2]))));
            return SMPSum(
                nt.firstTag().params[1],
                index,
                nt.firstTag().params[1],
                ScatQuestionMark(fTensor(
                    fBase(index),
                    fId(c[1].dims()[1])
                ))
                * c[1] * myCross
            );
        end
    ),
#D            applicable :=(self,t) >> FirstTagEq(t, AParSMP) and t.params[3][1][1]<>0 and
#D               t.params[2][t.params[3][1][1]]=APar and
#D               t.params[4][t.params[3][1][1]]=GetFirstTag(t).params[1] and
#D               t.params[3][1][1]=1, #t.params[3][1][1]=1 is a trick to prevent // the 2nd input because TensorGeneral breaks at the moment
#D            freedoms := nt -> [[1]],
#D            child := function (nt,freedoms)
#D              local PV,sizes;
#D              sizes:=Copy(nt.params[4]);
#D              sizes[nt.params[3][1][1]]:=1;
#D              if Length(Filtered(sizes, t->t<>1))>0 then
#D                  PV:=Copy(nt.params[2]);
#D                  PV[nt.params[3][1][1]]:=AOne;
#D              return [TTensorI_OL(DropParTag(nt.params[1]),
#D                          PV,nt.params[3],sizes,Drop(nt.params[5],1))];
#D              else
#D                  return [DropParTag(Copy(nt.params[1]))];
#D              fi;
#D            end,
#D            apply := function(nt,c,cnt)
#D              local myCross,a,b,index;
#D              index:=Ind(GetFirstTag(nt).params[1]);
#D              a:=List(c[1].dims()[2],x->fId(x));
#D              a[nt.params[3][1][1]]:=fTensor(fBase(index),a[nt.params[3][1][1]]);
#D              b:=List(a,x->Gath(x));
#D              myCross:=Cross(b);
#D#              return SMPSum(GetFirstTag(nt).params[1],index, GetFirstTag(nt).params[1],ScatQuestionMark(fTensor(fBase(index), fId(c[1].dims()[1]))) *c[1]* Cross(Gath(fTensor(fBase(index), fId(c[1].dims()[2][1]))),Gath(fId(c[1].dims()[2][2]))));
#D              return SMPSum(GetFirstTag(nt).params[1],index, GetFirstTag(nt).params[1],ScatQuestionMark(fTensor(fBase(index), fId(c[1].dims()[1]))) *c[1]* myCross);
#D            end),

    #That code vectorizes the last guy if it is a AVec of the vector size
    #Hack, only works with one output
#Changed by Marek, looks OK
    TTensorI_OL_Vectorize_AVecLast :=rec(
        applicable := (self,t) >>
            t.isTag(1, AVecReg)
            and Last(t.params[3][1])<>0
            and AVec = t.params[2][Last(t.params[3][1])]
            and t.params[4][Last(t.params[3][1])] = t.firstTag().v,

        freedoms := nt -> [[1]],

        child := function(nt, freedoms)
            local PV, sizes;

            sizes := Copy(nt.params[4]);
            sizes[Last(nt.params[3][1])] := 1;

            if Length(Filtered(sizes, t->t<>1))>0 then
                PV:=Copy(nt.params[2]);
                PV[Last(nt.params[3][1])]:=AOne;
                return [TTensorI_OL(
                    DropVectorTag(nt.params[1]).setWrap(VOLWrap(nt.firstTag().isa)),
                    PV,
                    nt.params[3],
                    sizes,
                    Drop(nt.params[5],1)
                )];
            else
                return [ DropVectorTag(Copy(nt.params[1])).setWrap(VOLWrap(nt.firstTag().isa)) ];
          fi;
        end,

        apply := function(nt,c,cnt)
            local myCross, myScat, mydims, v;

            v := nt.firstTag().v;

            #little hack for KernelDup
            if ObjId(nt.params[1]).name <> "KernelMMMDuped" then
                myCross := Cross(List(nt.dims()[2], t ->
                    VGath_dup(fId(t), v)
                ));
                myCross._children[Last(nt.params[3][1])] :=
                    VPrm_x_I(
                        fId( nt.dims()[2][Last(nt.params[3][1])] / v ),
                        nt.firstTag().v
                    );
            else
                mydims := nt.dims()[2];
                mydims[1] := mydims[1] * v;
                myCross := Cross(List(mydims, t ->
                    VPrm_x_I( fId(t/v), v )
                ));
            fi;

            myScat := VScat(fId( nt.dims()[1]/v ), v );

            return myScat * VTensor_OL(c[1], v) * myCross;
        end
    ),
#D            applicable :=(self,t) >> FirstTagEq(t, AVecReg) and Last(t.params[3][1])<>0 and t.params[2][Last(t.params[3][1])]=AVec and t.params[4][Last(t.params[3][1])]=GetFirstTag(t).v,
#D            freedoms := nt -> [[1]],
#D            child := function (nt,freedoms)
#D              local PV,sizes;
#D              sizes:=Copy(nt.params[4]);
#D              sizes[Last(nt.params[3][1])]:=1;
#D              if Length(Filtered(sizes, t->t<>1))>0 then
#D                  PV:=Copy(nt.params[2]);
#D                  PV[Last(nt.params[3][1])]:=AOne;
#D              return [TTensorI_OL(DropVectorTag(nt.params[1]).setWrap(VOLWrap(GetFirstTag(nt).isa)),
#D                          PV,nt.params[3],sizes,Drop(nt.params[5],1))];
#D              else
#D                  return [DropVectorTag(Copy(nt.params[1])).setWrap(VOLWrap(GetFirstTag(nt).isa))];
#D              fi;
#D            end,
#D            apply := function(nt,c,cnt)
#D              local myCross,myScat,mydims;
#D#little hack for KernelDup
#D              if (ObjId(nt.params[1]).name<>"KernelMMMDuped") then
#D                  myCross:=Cross(List(nt.dims()[2],t->VGath_dup(fId(t),GetFirstTag(nt).v)));
#D                  myCross._children[Last(nt.params[3][1])]:=
#D                  VPrm_x_I(fId(nt.dims()[2][Last(nt.params[3][1])]/GetFirstTag(nt).v),GetFirstTag(nt).v);
#D              else
#D                  mydims:=nt.dims()[2];
#D                  mydims[1]:=mydims[1]*GetFirstTag(nt).v;
#D                  myCross:=Cross(List(mydims,t->VPrm_x_I(fId(t/GetFirstTag(nt).v),GetFirstTag(nt).v)));
#D              fi;
#D              myScat:=VScat(fId(nt.dims()[1]/GetFirstTag(nt).v),GetFirstTag(nt).v);
#D              return myScat*VTensor_OL(c[1],GetFirstTag(nt).v)*myCross;
#D            end),

#Changed by Marek, looks OK
    TTensorI_OL_Vectorize_AParFirst :=rec(
        switch:=false,

        applicable := (self,t) >>
            t.isTag(1, AVecReg)
            and 0 <> First(t.params[3][1], x -> true)
            and APar = t.params[2][First(t.params[3][1],x -> true)]
            and t.firstTag().v = t.params[4][First(t.params[3][1], x -> true)],

        freedoms := nt -> [[1]],

        child := function (nt,freedoms)
            local PV,outorder,theguy;

            PV := Copy(nt.params[2]);
            PV[First(nt.params[3][1],x->true)] := AVec;

            theguy := Copy(First(nt.params[3][1], x->true));
            outorder := Drop(Copy(nt.params[3][1]),1);
            Add(outorder,theguy);

            return [
                TTensorI_OL( nt.params[1], PV, [outorder], nt.params[4], nt.params[5]),
                TL(nt.rng()[1].size, nt.firstTag().v, 1, 1, nt.params[5]),
                TL(
                    nt.dmn()[First(nt.params[3][1], x -> true)].size,
                    nt.dmn()[First(nt.params[3][1], x -> true)].size / nt.firstTag().v,
                    1,
                    1,
                    nt.params[5]
                )
            ];
        end,

        apply := function(nt,c,cnt)
            local i;

            i := Identities(nt.dmn());
            i._children[First(nt.params[3][1], x -> true)] := c[3];

            return c[2] * c[1] * i;
        end
    ),

#D            applicable :=(self,t) >> FirstTagEq(t, AVecReg) and First(t.params[3][1],x->true)<>0 and t.params[2][First(t.params[3][1],x->true)]=APar and t.params[4][First(t.params[3][1],x->true)]=GetFirstTag(t).v,
#D            freedoms := nt -> [[1]],
#D            child := function (nt,freedoms)
#D              local PV,outorder,theguy;
#D              PV:=Copy(nt.params[2]);
#D              PV[First(nt.params[3][1],x->true)]:=AVec;
#D              theguy:=Copy(First(nt.params[3][1],x->true));
#D              outorder:=Drop(Copy(nt.params[3][1]),1);
#D              Add(outorder,theguy);
#D              return [TTensorI_OL(nt.params[1],PV,[outorder],nt.params[4],nt.params[5]),TL(nt.rng()[1].size,GetFirstTag(nt).v,1,1,nt.params[5]),TL(nt.dmn()[First(nt.params[3][1],x->true)].size,nt.dmn()[First(nt.params[3][1],x->true)].size/GetFirstTag(nt).v,1,1,nt.params[5])];
#D            end,
#D            apply := function(nt,c,cnt)
#D              local i;
#D              i:=Identities(nt.dmn());
#D              i._children[First(nt.params[3][1],x->true)]:=c[3];
#D              return c[2]*c[1]*i;
#D            end),
));


# Copyright (c) 2018-2021, Carnegie Mellon University
# See LICENSE for details

_sum2 := function(lst)
    local i, res;
    res := 0;
    for i in [2..Length(lst)] do
        res := res + lst[i][1] + lst[i][2];
    od;
    return res;
end;

# objhead = [ objid_addr, param1_uid, param2_uid, ... ]
# uid is unique position in the hashtable returned by HashAdd
Class(ObjHashBase, HashTable(
    (objhead,size) -> When(IsInt(objhead), 1 + (objhead mod size),
                       1 + ((objhead[1] + _sum2(objhead)) mod size)),
    (objhead1, objhead2) -> objhead1 = objhead2)
);

ObjHashBase.operations := WithBases(HashOps, rec(
    Print := o -> Print(o.name)
));

Class(ObjHash, ObjHashBase, rec(
    liveEntries := self >> Filtered(self.entries, True),
    numLiveEntries := self >> Sum(List(self.liveEntries(), Length)),

    nonrecLookupAdd := meth(self, o)
        local lkup;
	lkup := HashLookupUID(self, InternalHash(o));
	if Same(lkup, false) then
	    return HashAdd(self, InternalHash(o), o);
	else return lkup;
	fi;
    end,

    listLookupAdd := meth(self, o)
        local lkup, uids;
	uids := [T_LIST];
	Append(uids, List(o, e -> self.uidObj(e)));
	lkup := HashLookupUID(self, uids);
	if Same(lkup, false) then
	    return HashAdd(self, uids, o);
	else return lkup;
	fi;
    end,

    uidObj := (self,o) >> Cond(
    IsRec(o) and IsBound(o.uid), o.uid,
    IsRec(o), Error("Unhashed object <o>"),
    IsList(o), self.listLookupAdd(o),
    self.nonrecLookupAdd(o)),

    objAdd := meth(self, res, h)
        local uid;
    uid := HashAdd(self, h, res);
    res.h := h;
    res.uid := uid;
    return res;
    end,

    singletonAdd := (self, o) >> self.objAdd(o, [BagAddr(o)]),

    _prList := meth(self, lst)
        local p;
    for p in lst do
            if IsRec(p) then Print(ObjId(p));
        elif IsList(p) then Print("(",self._prList(p), ")");
        else Print(p);
        fi;
        Print(":",self.uidObj(p), " ");
        od;
    end,

    debug := true,

    objLookup := meth(self, objid, params)
        local h,lkup,hcodes;
	h := [BagAddr(objid)];
	Append(h, List(params, p -> self.uidObj(p)));
	lkup := HashLookup(self, h);

	if self.debug then
	    if Same(lkup,false) then
		Print("(", objid, " ", self._prList(params), ")\n");
	    else
		Print(objid, " : hit ", lkup.uid, "\n");
	    fi;
	fi;

	return [lkup,h];
    end,

    memClassFunc := meth(self, cls, orig, bck)
        Constraint(IsBound(cls.(orig)));
        #Constraint(not IsBound(cls.__call_no_memo__));
    cls.(bck) := cls.(orig);
    cls._hash := self;
    bck := RecName(bck); # this will make lookup a bit faster

    cls.(orig) := meth(arg)
            local clsself, params, lkup, res, h;
        clsself := arg[1]; params := Drop(arg,1);
        h := clsself._hash;
        if h<>false then
        lkup := h.objLookup(clsself, params);
        if lkup[1] <> false then return lkup[1]; fi;
        fi;
        res := ApplyFunc(clsself.(bck), params);
        if h<>false then
        return h.objAdd(res, lkup[2]);
        else return res;
        fi;
    end;
    end,

    memClass := (self,cls) >> self.memClassFunc(cls, "__call__", "__call_no_memo__")
));


# Copyright (c) 2018-2021, Carnegie Mellon University
# See LICENSE for details

Declare(SSEUnparser);

_toReal := v -> Cond(IsValue(v), Value(TReal, v.v), tcast(TReal, v));

@Value      := @.cond(IsValue);
@TInt       := @.cond(x->IsIntT(x.t));
@TReal      := @.cond(x->IsRealT(x.t));
@TRealInt   := @.cond(x->IsIntT(x.t) or IsRealT(x.t));
@_scalar    := @.cond(x->IsIntT(x.t) or IsRealT(x.t) or IsPtrT(x.t));
@TVect      := @.cond(x->IsVecT(x.t));
@TVectUChar := @.cond(x->IsVecT(x.t) and ObjId(x.t.t)=TUChar);

_isa := self -> self.opts.vector.isa;
_epi_or_px := (self, o) -> When(_isa(self).isFixedPoint or IsOrdT(o.t.t),
    "epi" :: self.ctype_suffixval(o.t, _isa(self)),
    self.ctype_suffix(o.t, _isa(self)));

_epu_or_px := (self, o) -> When(_isa(self).isFixedPoint,
    "epu" :: self.ctype_suffixval(o.t, _isa(self)),
    self.ctype_suffix(o.t, _isa(self)));

_epi := (self, o) -> Concat("epi", self.ctype_suffixval(o.t, _isa(self)));

_epu_to_epi := (s) -> Cond( s{[1..3]} = "epu", "epi" :: s{[4..Length(s)]}, s );

Class(SSEUnparser, CMacroUnparserProg, rec(
    # -----------------------------
    # ISA independent constructs
    # -----------------------------
    nth := (self, o, i, is) >> self.printf("$1[$2]", [o.loc, o.idx]),
    fdiv := (self, o, i, is) >> self.printf("(((double)$1) / ($2))", o.args),
    div  := (self, o, i, is) >> self.printf("(($1) / ($2))", o.args),
    idiv := (self, o, i, is) >> self.printf("(($1) / ($2))", o.args),

    # --------------------------------
    # ISA constructs, general
    # -------------------------------

    # This is a general suffix for intrinsics that is determine from the data type
    ctype_suffix := (self, t, isa) >> Cond(
        t = TVect(T_Int(128), 1),  "epi128",
        t = TVect(T_UInt(128), 1), "epu128",

        t = TVect(T_Real(32), 4) or
        t = TVect(T_Real(32), 2) and isa=SSE_2x32f or
        t = TVect(TReal, 4) and isa=SSE_4x32f or
        t = TVect(TReal, 2) and isa=SSE_2x32f,
        "ps",

        # no way to create __m64 type directly from floats, have to go thru integers
        t = TVect(T_Real(32), 2) and isa=SSE_4x32f or
        t = TVect(TReal, 2) and isa=SSE_4x32f,
        "ps_half",

        t = TVect(TInt, 4) or
        t = TVect(T_Int(32), 4) or
        t = TVect(TReal, 4) and isa.isFixedPoint,
        "epi32",

        t = TVect(T_UInt(32), 4), "epu32",

        t = TVect(T_Real(64), 2) or
        t = TVect(TReal, 2) and isa=SSE_2x64f,
        "pd",

        t = TVect(TInt, 2) or
        t = TVect(T_Int(64), 2) or
        t = TVect(TReal, 2) and isa.isFixedPoint,
        "epi64",

        t = TVect(T_UInt(64), 2), "epu64",

        t = TVect(T_Int(16),  8), "epi16",
        t = TVect(T_UInt(16), 8), "epu16",
        t = TVect(TInt,       8), "epi16",
        t = TVect(TReal,      8), "epi16",

        t = TVect(T_Int(8),  16), "epi8",
        t = TVect(T_UInt(8), 16), "epu8",
        t = TVect(TReal,     16), When(isa.isSigned, "epi8", "epu8"),
        t = TVect(TInt,      16), "epi8",
        t = TVect(TUChar,    16), "epu8",
        ""
    ),

    mul_suffix := (t,isa) -> Cond(
        t = TVect(T_Real(32), 2), "_ps",
        t = TVect(T_Real(32), 4), "_ps",
        t = TVect(T_Real(64), 2), "_pd",
        t = TVect(TReal, 2) and isa = SSE_2x32f, "_ps",
        t = TVect(TReal, 2), "_pd",
        t = TVect(TInt, 2), "_epi64",
        t = TVect(TReal, 4), "_ps",
        t = TVect(TInt, 4), "_epi32",
        t = TVect(T_Int(32), 4), "lo_epi32",
        t = TVect(TReal, 8), "lo_epi16",
        t = TVect(TReal, 16), Error("16-way multiplication is not supported"),
        t = TVect(TUChar, 16), Error("16-way multiplication is not supported"),
        ""
    ),

    # This is a general suffix for intrinsics that is determine from the data type
    ctype_suffixval := (t, isa) -> Cond(
	t = TVect(TReal, 2), "64",
	t = TVect(TInt, 4), "32",
	t = TVect(TReal, 4), "32",
	t = TVect(T_Int(32), 4), "32",
    t = TVect(T_UInt(32), 4), "32",
	t = TVect(TReal, 8), "16",
	t = TVect(TInt, 8), "16",
	t = TVect(T_Int(16), 8), "16",
    t = TVect(T_UInt(16), 8), "16",
	t = TVect(TReal, 16), "8",
	t = TVect(TInt, 16), "8",
	t = TVect(T_Int(8), 16), "8",
    t = TVect(T_UInt(8), 16), "8",
	t = TVect(TUChar, 16), "8",
	t = TVect(T_Real(32), 4), "32",
	t = TVect(T_Real(64), 2), "64",
	""
    ),

    # This is the type used for declarations of vector variables
    ctype := (self, t, isa) >> Cond(
        # NOTE: used for unaligned vector pointers,for single prec, it should be "float"
	t in [TReal, TVect(TReal, 1)],
            Cond(isa = SSE_2x64f, "double",
		 isa = SSE_2x64i, "__int64",
		 isa = SSE_4x32f, "float",
		 isa = SSE_2x32f, "float",
		 isa = SSE_4x32i, "__int32",
		 isa = SSE_8x16i, "short",
		 isa = SSE_16x8i, Cond(isa.isSigned, "char", "unsigned char"),
		 isa.ctype),

	t = TVect(TReal, 2),
            Cond(isa = SSE_2x64f, "__m128d",
		 isa = SSE_2x64i, "__m128i",
		 isa = SSE_4x32f, "__m64",
		 isa = SSE_8x16i, "__int32",
		 isa = SSE_16x8i, "__int16",
		 isa = SSE_2x32f, "__m64"),

	t = TVect(TReal, 4),
            Cond(isa = SSE_4x32f, "__m128",
		 isa = SSE_2x32f, "__m128",
		 isa = SSE_4x32i, "__m128i"),

        t = TVect(TInt,    2), "__m128i",
	t = TVect(TInt,    4), "__m128i",
	t = TVect(TInt,    8), "__m128i",
	t = TVect(TReal,   8), "__m128i",
	t = TVect(TInt,   16), "__m128i",
	t = TVect(TUChar, 16), "__m128i",
	t = TVect(TReal,  16), "__m128i",

	t = TInt,
            Cond(isa = SSE_2x64i, "__int64",
		 isa = SSE_4x32i, "__int32",
		 isa = SSE_8x16i, "short",
		 isa = SSE_16x8i, "char",
		 "int"),

	t = TVect(T_Int(128), 1), "__m128i",
	t = TVect(T_Int(64),  2), "__m128i",
	t = TVect(T_Int(32),  4), "__m128i",
	t = TVect(T_Int(16),  8), "__m128i",
	t = TVect(T_Int(8),  16), "__m128i",

	t = TVect(T_UInt(128), 1), "__m128i",
	t = TVect(T_UInt(64),  2), "__m128i",
	t = TVect(T_UInt(32),  4), "__m128i",
	t = TVect(T_UInt(16),  8), "__m128i",
	t = TVect(T_UInt(8),  16), "__m128i",

        t = TVect(T_Real(32), 2), "__m64",
	t = TVect(T_Real(32), 4), "__m128",
	t = TVect(T_Real(64), 2), "__m128d",
	Error(self,".ctype doesn't know type ",t)
    ),

    cvalue_suffix  := (self, t)  >> let( isa := _isa(self), Cond(
        (t = TReal and isa in [SSE_2x32f, SSE_4x32f]) or t = T_Real(32), "f",
        (t = TReal) or t = T_Real(64), "",
        Error(self,".cvalue_suffix doesn't know type ",t)
    )),

    vhex := (self, o, i, is) >> Print("_mm_set_", _epi(self, o), "(", self.infix(Reversed(o.p), ", "), ")"),

    Value := (self, o, i, is) >> let(zero := "0" :: self.cvalue_suffix(TReal), Cond(
        o.t = TString, Print(o.v),

        o.t = TReal or ObjId(o.t)=T_Real, let(v := When(IsCyc(o.v), ReComplex(Complex(o.v)), Double(o.v)),
            When(v<0, Print("(", v, self.cvalue_suffix(o.t), ")"), Print(v, self.cvalue_suffix(o.t)))),

        #IsComplexT(o.t),
	#    Print("COMPLEX(", ReComplex(Complex(o.v)), self.cvalue_suffix(o.t.realType()), ", ",
	#        ImComplex(Complex(o.v)), self.cvalue_suffix(o.t.realType()), ")"),

        IsIntT(o.t) or IsUIntT(o.t),
            When(o.v < 0, Print("(", o.v, ")"), Print(o.v)),

        ObjId(o.t) = TVect and _isa(self) = SSE_2x32f,
            Cond(self.cx.isInside(Value) and Length(self.cx.Value) >= 2, # nested in an array
		 Print(          "{", zero, ", ", zero, ", ", self.infix((o.v), ", "), "}"),
		 Print("_mm_set_ps(", zero, ", ", zero, ", ", self.infix(Reversed(o.v), ", "), ")")),

        ObjId(o.t) = TVect and Length(Set(o.v)) = 1,
            Cond(self.cx.isInside(Value) and Length(self.cx.Value) >= 2, # nested in an array
		 Print("{", self.infix(Replicate(o.t.size, o.v[1]), ", "), "}"),
		 Print("_mm_set1_", _epi_or_px(self, o), "(", self(o.v[1], i, is), ")")),

        ObjId(o.t) = TVect,
            Cond(self.cx.isInside(Value) and Length(self.cx.Value) >= 2, # nested in an array
		 Print(                                 "{", self.infix((o.v), ", "), "}"),
		 Print("_mm_set_", _epi_or_px(self, o), "(", self.infix(Reversed(o.v), ", "), ")")),

        IsArray(o.t),
            Print("{", self.infix(o.v, ", "), "}"),

        ObjId(o.t) = TSym,
            Print("(", self.declare(o.t, [], 0, 0), ") ", o.v),

        o.t = TBool, Print(When(o.v = true, "1", "0")),

	Inherited(o, i, is)
    )),

    vpack := (self, o, i, is) >> let(
        sfx := _epi_or_px(self, o),
        Print("_mm_set_", sfx, "(", self.infix(Reversed(o.args), ", "), ")")),

    vdup := (self, o, i, is) >> let(
	sfx := _epi_or_px(self, o),
        CondPat(o,
            [vdup, nth, @.cond(x->x.t=TInt and x.v=2)], self.printf("_mm_loaddup_$1(&($2))", [sfx, o.args[1]]),
            [vdup, @, @TInt], self.printf("_mm_set1_$1($2)", [sfx, o.args[1]]))),

    # --------------------------------
    # Declarations
    _declTVect := (self, t, vars, i, is) >> let(ctype := self.ctype(t, _isa(self)), Print(ctype, " ", self.infix(vars, ", ", i+is))),
    _unparseTVect := (self, t, i, is) >> let(ctype := self.ctype(t, _isa(self)), Print(ctype)),

    TVect := arg >> When(Length(arg)=5, arg[1]._declTVect(arg[2], arg[3], arg[4], arg[5]),
	                                arg[1]._unparseTVect(arg[2], arg[3], arg[4])),
    TReal := ~.TVect,
    TInt  := (self, t, vars, i, is) >> Print("int ", self.infix(vars, ", ", i+is)),
    TBool := (self, t, vars, i, is) >> Print("int ", self.infix(vars, ", ", i+is)),

    # --------------------------------
    # Arithmetic
    #
    mul := (self, o, i, is) >> let(n := Length(o.args), Cond(
        not IsVecT(o.t),
            Print("(",self.pinfix(o.args, ")*("),")"),
	n > 2 and n mod 2 <> 0,
            self(mul(o.args[1], ApplyFunc(mul, Drop(o.args, 1))), i, is),
        n > 2,
            self(mul(ApplyFunc(mul, o.args{[1..n/2]}), ApplyFunc(mul, o.args{[n/2+1..n]})), i, is),
        CondPat(o,
	    [mul, @TReal, @TVect], Cond(_isa(self) = SSE_2x32f,
                self(mul(vdup(o.args[1], 4), o.args[2]), i, is), # NOTE: HACK for SSE_2x32f
                self(mul(vdup(o.args[1], o.t.size), o.args[2]), i, is)),
	    [mul, @TVect, @TReal],  self(mul(o.args[1], vdup(o.args[2],o.t.size)), i, is),
            # NOTE: This hack is probably no longer necessary (was used for PRDFTs)
	    [mul, @(1, cond, e -> e.t=TInt), @TVect],
	        self(mul(cond(o.args[1].args[1],
			      vdup(o.t.t.value(o.args[1].args[2]), o.t.size),
			      vdup(o.t.t.value(o.args[1].args[3]), o.t.size)), o.args[2]), i, is),
	    [mul, @TInt, @TVect],  self(mul(vdup(_toReal(o.args[1]),o.t.size), o.args[2]), i, is),
	    [mul, @TVect, @TInt],  self(mul(o.args[1], vdup(_toReal(o.args[2]),o.t.size)), i, is),
	    [mul, @TVect, @TVect], self.printf("_mm_mul$1($2, $3)", [self.mul_suffix(o.t, _isa(self)), o.args[1], o.args[2]]),
	    Error("Don't know how to unparse <o>. Unrecognized type combination")
    ))),

    fpmul := (self, o, i, is) >> let(isa := _isa(self), CondPat(o,
        # preparing for SSSE3 _mm_mulhrs_epi16 (__m128i a, __m128i b)
        # self.printf("_mm_mulhrs_epi16($1, $2)", [o.args[2], o.args[3].t.value(List(o.args[3].v, i->bin_shl(i,1)))]),
        [fpmul, @, @TVect, @], self.printf("$1($2($3, $4), $5)",
	    [isa.vlshift, isa.vmul, o.args[2], o.args[3], isa.bits-o.args[1]]),

        [fpmul, @, @, @],      self.printf("$1($2(_mm_set1_$3($4), $5), $6)",
	    [isa.vlshift, isa.vmul, self.ctype_suffix(o.t, _isa(self)), o.args[2], o.args[3], isa.bits-o.args[1]]))),

    add := (self, o, i, is) >> let(n := Length(o.args), Cond(
	not IsVecT(o.t),
            self.pinfix(o.args, " + "),
        n > 2 and n mod 2 <> 0,
            self(add(o.args[1], ApplyFunc(add, Drop(o.args, 1))), i, is),
        n > 2,
            self(add(ApplyFunc(add, o.args{[1..n/2]}), ApplyFunc(add, o.args{[n/2+1..n]})), i, is),
        let(isa := _isa(self), # ugly, backward compatibility, use <adds> instead
	    saturated := When(IsBound(isa.isFloat) and IsBound(isa.saturatedArithmetic) and not isa.isFloat and isa.saturatedArithmetic, "s", ""),
	    _sfx      := self.ctype_suffix(o.t, isa),
	    sfx       := Cond( saturated="", _epu_to_epi(_sfx), _sfx),
	    CondPat(o,
		[add, @TVect,   @TVect], self.printf("_mm_add$1_$2($3, $4)", [saturated, sfx, o.args[1], o.args[2]]),
		Error("Don't know how to unparse <o>. Unrecognized type combination"))))),

    adds := (self, o, i, is) >> CondPat(o,
		[adds, @TVect, @TVect, ...],
		    Cond( Length(o.args)>2,
		        self(adds(o.args[1], brackets(ApplyFunc(adds, Drop(o.args, 1)))), i, is),
		        self.printf("_mm_adds_$1($2, $3)", [self.ctype_suffix(o.t, rec()), o.args[1], o.args[2]])),
		Inherited(o, i, is)),

    _sub := (self, t, a, i, is) >> let(
	isa := _isa(self),
	sfx := _epu_to_epi(self.ctype_suffix(t, isa)),
	saturated := When(IsBound(isa.isFloat) and IsBound(isa.saturatedArithmetic) and not isa.isFloat and isa.saturatedArithmetic, "s", ""),
	CondPat(a,
            [ListClass, @TVect,   @TVect], self.printf("_mm_sub$1_$2($3, $4)", [saturated, sfx, a[1], a[2]]),
            [ListClass, @, @],             self.printf("($1 - ($2))", a),
            Error("Don't know how to unparse subtraction of a[1] and a[2]. Unrecognized type combination"))),

    sub := (self, o, i, is) >> self._sub(o.t, o.args, i, is),

    neg := (self, o, i, is) >> CondPat(o,
        [@, @TVect], self._sub(o.t, [o.t.zero(), o.args[1]], i, is),
        self.printf("(-$1)", o.args)),

    stickyNeg := ~.neg,

    sqrt  := (self, o, i, is) >> Cond( IsVecT(o.t),
        Checked( IsRealT(o.t.t), self.printf("_mm_sqrt_$1($2)", [self.ctype_suffix(o.t, _isa(self)), o.args[1]])),
        Inherited(o, i, is)),

    rsqrt := (self, o, i, is) >> Cond( IsVecT(o.t), let( sfx := self.ctype_suffix(o.t, _isa(self)),
        Checked( sfx="ps", self.printf("_mm_rsqrt_ps($1)", [o.args[1]]))),
        Inherited(o, i, is)),

    # assuming we have ICC <ia32intrin.h> here
    log := (self, o, i, is) >> Cond( IsVecT(o.t), let( sfx := self.ctype_suffix(o.t, _isa(self)),
        Checked( sfx in ["ps", "pd"], Cond(
            Length(o.args)>1 and (o.args[2]=2 or o.args[2]=o.t.value(2)),
                self.printf("_mm_log2_$1($2)", [sfx, o.args[1]]),
            Length(o.args)>1 and (o.args[2]=10 or o.args[2]=o.t.value(10)),
                self.printf("_mm_log10_$1($2)", [sfx, o.args[1]]),
            Length(o.args)=1 or o.args[2]=d_exp(1) or o.args[2]=o.t.value(d_exp(1)),
                self.printf("_mm_log_$1($2)", [sfx, o.args[1]]),
            self.printf("_mm_div_$1(_mm_log_$1($2), _mm_log_$1($3))", [sfx, o.args[1]])))),
        Inherited(o, i, is)),

    # assuming we have ICC <ia32intrin.h> here
    exp := (self, o, i, is) >> Cond( IsVecT(o.t), let( sfx := self.ctype_suffix(o.t, _isa(self)),
        Checked( sfx in ["ps", "pd"], self.printf("_mm_exp_$1($2)", [sfx, o.args[1]]))),
        Inherited(o, i, is)),

    # assuming we have ICC <ia32intrin.h> here
    pow := (self, o, i, is) >> Cond( IsVecT(o.t), let( sfx := self.ctype_suffix(o.t, _isa(self)),
        Checked( sfx in ["ps", "pd"], Cond(
            o.args[1]=2 or o.args[1]=o.t.value(2),
                self.printf("_mm_exp2_$1($2)", [sfx, o.args[2]]),
            o.args[1]=d_exp(1) or o.args[1]=o.t.value(d_exp(1)),
                self.printf("_mm_exp_$1($2)", [sfx, o.args[2]]),
            self.printf("_mm_pow_$1($2, $3)", [sfx, o.args[1], o.args[2]])))),
        Inherited(o, i, is)),

    imod  := (self, o, i, is) >> Cond( IsIntT(o.t.base_t()) and Is2Power(o.args[2]),
        # in two's complement arithmetics this will work for both positive and negative o.args[1]
        self(bin_and(o.args[1], o.args[2]-1), i, is),
        self.printf("(($1) % ($2))", o.args)),

    # --------------------------------
    # logic
    #
    arith_shl := (self, o, i, is) >> self.prefix(_isa(self).vlshift, o.args),
    arith_shr := (self, o, i, is) >> CondPat( o,
        [arith_shr, @.cond(x->x.t=TVect(T_Int(32), 4)), @],
            self.prefix("_mm_srai_epi32", o.args),
        [arith_shr, @.cond(x->x.t=TVect(T_Int(16), 8)), @],
            self.prefix("_mm_srai_epi16", o.args),
        [arith_shr, @TVect, @],
            self.prefix(_isa(self).vrshift, o.args),
        Inherited(o, i, is)),

    bin_xor := (self, o, i, is) >> CondPat(o,
                [bin_xor, @TVect, @TVect], self.prefix("_mm_xor_si128", o.args),
                Inherited(o, i, is)),
    bin_and := (self, o, i, is) >> CondPat(o,
                [bin_and, @TVect, @TVect], self.prefix("_mm_and_si128", o.args),
                Inherited(o, i, is)),

    bin_andnot := (self, o, i, is) >> self.prefix("_mm_andnot_si128", o.args),

    bin_or := (self, o, i, is) >> CondPat(o,
        [bin_or, @TVect, @TVect], let(sfx := self.ctype_suffix(o.t, _isa(self)),
	    Cond( not (sfx in ["ps", "pd", "ps_half"]), #was: _isa(self).isFixedPoint,
		self.printf("_mm_or_si128($1, $2)", o.args),
		self.printf("_mm_castsi128_$3(_mm_or_si128(_mm_cast$3_si128($1), _mm_cast$3_si128($2)))",
		            o.args :: [sfx]))),
        [bin_or, @TReal, @TReal], self.printf("(($1) | ($2))", o.args),

	Inherited(o, i, is)),

    min := (self, o, i, is) >> CondPat(o,
        [min, @TVect, @TVect], self.prefix("_mm_min_" :: self.ctype_suffix(o.t, _isa(self)), o.args),
            Inherited(o, i, is)),

    max := (self, o, i, is) >> let(n := Length(o.args), When(
    	IsVecT(o.t) and n > 2, self.printf("_mm_max_$1($2, $3)", [self.ctype_suffix(o.t, _isa(self)), o.args[1], ApplyFunc(max, Drop(o.args, 1))]),
        CondPat(o,
            [max, @TVect, @TVect], self.prefix("_mm_max_" :: self.ctype_suffix(o.t, _isa(self)), o.args),
                Inherited(o, i, is)))),

    abs := (self, o, i, is) >> CondPat(o,
        [abs, @TVect], let( sfx := self.ctype_suffix(o.t, _isa(self)),
                            Cond( sfx = "ps", self.printf("_mm_castsi128_ps(_mm_and_si128(_mm_castps_si128($1), _mm_set_epi32(0x7FFFFFFF, 0x7FFFFFFF, 0x7FFFFFFF, 0x7FFFFFFF)))", o.args),
                                  sfx = "pd", self.printf("_mm_castsi128_pd(_mm_and_si128(_mm_castpd_si128($1), _mm_set_epi32(0x7FFFFFFF, 0xFFFFFFFF, 0x7FFFFFFF, 0xFFFFFFFF)))", o.args),
                                  Error("not implemented"))),
            Inherited(o, i, is)),

    bin_shl := (self, o, i, is) >> CondPat(o,
        [bin_shl, @TVect, @TInt], let(
            sfx := self.ctype_suffix(o.t, _isa(self)),
	    Cond( _isa(self).isFixedPoint,                            # legacy
		     self.printf("_mm_slli_si128($1, $2)", o.args),   # legacy
		  sfx in ["epi16", "epi32", "epi64", "epu16", "epu32", "epu64"],
		      self.printf("_mm_slli_$3($1, $2)", o.args :: [_epu_to_epi(sfx)]),
		  sfx in ["epi8", "epu8"],
		      Error("bin_shl is undefined for epi8 and epu8"),
		  sfx in ["epi128", "epu128"], # shift with byte granularity
		      self.printf("_mm_slli_si128($1, $2)", [o.args[1], idiv(o.args[2], 8)] ),
		  # else, shift whole register with shift argument specified in bytes (legacy, fix using epi128 in ISAs first)
		  self.printf("_mm_castsi128_$3(_mm_slli_si128(_mm_cast$3_si128($1), $2))", o.args :: [sfx]))),
        [bin_shl, @TReal, @TInt], self.printf("(($1) << ($2))", o.args),
        [bin_shl, @TInt, @TInt], self.printf("(($1) << ($2))", o.args),
        [bin_shl, @, @], self.prefix("_mm_slli_" :: self.ctype_suffix(o.t, _isa(self)), o.args)),

    bin_shr := (self, o, i, is) >> CondPat(o,
        [bin_shr, @TVect, @TInt], let(
            sfx := self.ctype_suffix(o.t, _isa(self)),
	    Cond( _isa(self).isFixedPoint,                            # legacy
		      self.printf("_mm_srli_si128($1, $2)", o.args),  # legacy
		  sfx in ["epi16", "epi32", "epi64", "epu16", "epu32", "epu64"],
		      self.printf("_mm_srli_$3($1, $2)", o.args :: [_epu_to_epi(sfx)]),
		  sfx in ["epi8", "epu8"],
		      Error("bin_shr is undefined for epi8 and epu8"),
		  sfx in ["epi128", "epu128"], # shift with byte granularity
		      self.printf("_mm_srli_si128($1, $2)", [o.args[1], idiv(o.args[2], 8)] ),
		  # else, shift whole register with shift argument specified in bytes (legacy, fix using epi128 in ISAs first)
		  self.printf("_mm_castsi128_$3(_mm_srli_si128(_mm_cast$3_si128($1), $2))", o.args :: [sfx]))),
	# default
	[bin_shr, @TReal, @TInt], self.printf("(($1) >> ($2))", o.args),
	[bin_shr, @TInt, @TInt], self.printf("(($1) >> ($2))", o.args),
	# what's this?
	[bin_shr, @, @], self.prefix("_mm_srli_" :: self.ctype_suffix(o.t, _isa(self)), o.args)),

    # vector shifts
    vec_shr := (self, o, i, is) >> let(
        isa := _isa(self),
        sfx := self.ctype_suffix(o.t, isa),
        # making sure this is SSE data type
        t   := Checked(IsVecT(o.t) and sfx<>"", o.t),
        a   := o.args[1],
        s   := o.args[2] * 16 / t.size,
        # may need typecasts to please compiler
        Cond( self.ctype(t, isa)="__m128i",
            self.printf("_mm_srli_si128($1, $2)", [a, s] ),
            self.printf("_mm_castsi128_$3(_mm_srli_si128(_mm_cast$3_si128($1), $2))", [a, s, sfx])
        )),

    vec_shl := (self, o, i, is) >> let(
        isa := _isa(self),
        sfx := self.ctype_suffix(o.t, isa),
        # making sure this is SSE data type
        t   := Checked(IsVecT(o.t) and sfx<>"", o.t),
        a   := o.args[1],
        s   := o.args[2] * 16 / t.size,
        # may need typecasts to please compiler
        Cond( self.ctype(t, isa)="__m128i",
            self.printf("_mm_slli_si128($1, $2)", [a, s] ),
            self.printf("_mm_castsi128_$3(_mm_slli_si128(_mm_cast$3_si128($1), $2))", [a, s, sfx])
        )),

    # --------------------------------
    # comparison
    #
    eq := (self, o, i, is) >> let( ctype := self.ctype_suffix(o.args[1].t, _isa(self)),
        sfx := _epu_to_epi(ctype),
        Cond(IsVecT(o.t), self.prefix("_mm_cmpeq_" :: sfx, o.args),
            Inherited(o, i, is))),

    lt := (self, o, i, is) >> Cond(IsVecT(o.t),
        self.prefix("_mm_cmplt_" :: self.ctype_suffix(o.args[1].t, _isa(self)), o.args),
        Inherited(o, i, is)),

    gt := (self, o, i, is) >> Cond(ObjId(o.t)=TVect,
        self.prefix("_mm_cmpgt_" :: self.ctype_suffix(o.args[1].t, _isa(self)), o.args),
        Inherited(o, i, is)),

    mask_eq := ~.eq,
    mask_lt := ~.lt,
    mask_gt := ~.gt,

    vparam := (self, o, i, is) >> iclshuffle(o.p),

    # --------------------------------
    # ISA specific : SSE_2x64f
    #
    vunpacklo_2x64f := (self, o, i, is) >> self.prefix("_mm_unpacklo_pd", o.args),
    vunpackhi_2x64f := (self, o, i, is) >> self.prefix("_mm_unpackhi_pd", o.args),
    vshuffle_2x64f  := (self, o, i, is) >> self.prefix("_mm_shuffle_pd", o.args),
    vushuffle_2x64f := (self, o, i, is) >> self(o.binop(o.args[1], o.args[1], o.args[2]), i, is),

    vload1sd_2x64f := (self, o, i, is) >> self.prefix("_mm_load_sd", o.args),
    vload_1l_2x64f := (self, o, i, is) >> self.prefix("_mm_loadl_pd", o.args),
    vload_1h_2x64f := (self, o, i, is) >> self.prefix("_mm_loadh_pd", o.args),
    vloadu_2x64f   := (self, o, i, is) >> self.prefix("_mm_loadu_pd", o.args),

    vstore_1l_2x64f := (self, o, i, is) >> Print(Blanks(i), self.prefix("_mm_storel_pd", o.args), ";\n"),
    vstore_1h_2x64f := (self, o, i, is) >> Print(Blanks(i), self.prefix("_mm_storeh_pd", o.args), ";\n"),
    vstoreu_2x64f   := (self, o, i, is) >> Print(Blanks(i), self.prefix("_mm_storeu_pd", o.args), ";\n"),

    addsub_2x64f := (self, o, i, is) >> Checked(Length(o.args) = 2,
        CondPat(o,
           [addsub_2x64f, @TReal, @TVect], self(addsub_2x64f(vdup(o.args[1],o.t.size), o.args[2]), i, is),
           [addsub_2x64f, @TVect, @TReal], self(addsub_2x64f(o.args[1], vdup(o.args[2], o.t.size)), i, is),
           [addsub_2x64f, @TInt,  @TVect], self(addsub_2x64f(vdup(_toReal(o.args[1]), o.t.size), o.args[2]), i, is),
           [addsub_2x64f, @TVect, @TInt],  self(addsub_2x64f(o.args[1], vdup(_toReal(o.args[2]), o.t.size)), i, is),
           [addsub_2x64f, @TVect, @TVect], self.printf("_mm_addsub_pd($1, $2)", o.args),
           Error("Don't know how to unparse <o>. Unrecognized type combination")
    )),

    hadd_2x64f := (self, o, i, is) >> self.printf("_mm_hadd_pd($1, $2)", [o.args[1], o.args[2]]),

    chslo_2x64f := (self, o, i, is) >> self.printf(
	"_mm_castsi128_pd(_mm_xor_si128(_mm_castpd_si128($1), _mm_set_epi32(0, 0, 0x80000000, 0)))", o.args),
    chshi_2x64f := (self, o, i, is) >> self.printf(
	"_mm_castsi128_pd(_mm_xor_si128(_mm_castpd_si128($1), _mm_set_epi32(0x80000000, 0, 0, 0)))", o.args),
    chshi_4x32f := (self, o, i, is) >> self.printf(
	"_mm_castsi128_ps(_mm_xor_si128(_mm_castps_si128($1), _mm_set_epi32(0x80000000, 0, 0x80000000, 0)))", o.args),
    chslo_4x32f := (self, o, i, is) >> self.printf(
	"_mm_castsi128_ps(_mm_xor_si128(_mm_castps_si128($1), _mm_set_epi32(0, 0x80000000, 0, 0x80000000)))", o.args),

    vcvt_64f32f := (self, o, i, is) >> self.prefix("_mm_cvtps_pd", o.args),

    cmpge_2x64f := (self, o, i, is) >> self.prefix("_mm_cmpge_pd", o.args),

    cmple_2x64f := (self, o, i, is) >> self.prefix("_mm_cmple_pd", o.args),
    cmpeq_2x64f := (self, o, i, is) >> self.prefix("_mm_cmpeq_pd", o.args),

    # --------------------------------
    # ISA specific : SSE_2x32f
    #
    vunpacklo_2x32f := (self, o, i, is) >> self.prefix("_mm_unpacklo_ps", o.args),
    vunpackhi_2x32f := (self, o, i, is) >> self.prefix("_mm_unpackhi_ps", o.args),
    vshuffle_2x32f  := (self, o, i, is) >> self.prefix("_mm_shuffle_ps", o.args),
    vushuffle_2x32f := (self, o, i, is) >> self(o.binop(o.args[1], o.args[1], o.args[2]), i, is),
    vload_2x32f     := (self, o, i, is) >> self.prefix("_mm_loadl_pi", o.args),
    vstore_2x32f    := (self, o, i, is) >> Print(Blanks(i), self.prefix("_mm_storel_pi", o.args), ";\n"),
    vstoreu_2x32f   := (self, o, i, is) >> Print(Blanks(i), self.printf("_mm_storel_epi64($1, _mm_castps_si128($2));\n", o.args)),
    vloadu_2x32f    := (self, o, i, is) >> self.printf("_mm_castsi128_ps(_mm_loadl_epi64($1))", o.args),

    # --------------------------------
    # ISA specific : SSE_4x32f
    #
    prefix_cast := (self, prefix, t, o) >> Cond( self.ctype(t, _isa(self)) = self.ctype(o.t, _isa(self)), self.prefix(prefix, o.args),
                                             self(tcast(o.t, ApplyFunc(ObjId(o), List(o.args, a -> Cond(IsExp(a), tcast(t, a), a)))), 0, 1)),

    vunpacklo_4x32f := (self, o, i, is) >> self.prefix_cast("_mm_unpacklo_ps", TVect(T_Real(32), 4), o),
    vunpackhi_4x32f := (self, o, i, is) >> self.prefix_cast("_mm_unpackhi_ps", TVect(T_Real(32), 4), o),
    vshuffle_4x32f  := (self, o, i, is) >> self.prefix_cast("_mm_shuffle_ps", TVect(T_Real(32), 4), o),
    vushuffle_4x32f := (self, o, i, is) >> self(o.binop(o.args[1], o.args[1], o.args[2]), i, is),
    hadd_4x32f  := (self, o, i, is) >> self.printf("_mm_hadd_ps($1, $2)", [o.args[1], o.args[2]]),
    vldup_4x32f := (self, o, i, is) >> self.prefix("_mm_moveldup_ps", o.args),
    vhdup_4x32f := (self, o, i, is) >> self.prefix("_mm_movehdup_ps", o.args),

    vinsert_4x32f  := (self, o, i, is) >> self.printf(
	"_mm_castsi128_ps(_mm_insert_epi32(_mm_castps_si128($1), $2, $3))", [o.args[1], o.args[2], o.args[3].p-1]),
    vextract_4x32f := (self, o, i, is) >> Print(Blanks(i),
	self.printf("$1 = _mm_extract_ps($2, $3)", [deref(o.args[1]), o.args[2], o.args[3]-1]), ";\n"),

    vload1_4x32f   := (self, o, i, is) >> self.prefix("_mm_load_ss", o.args),
    vload_2l_4x32f := (self, o, i, is) >> self.prefix("_mm_loadl_pi", o.args),
    vload_2h_4x32f := (self, o, i, is) >> self.prefix("_mm_loadh_pi", o.args),
    vloadu_4x32f   := (self, o, i, is) >> self.printf("_mm_loadu_ps($1)", o.args),
    vloadu2_4x32f  := (self, o, i, is) >> self.printf("_mm_castsi128_ps(_mm_loadl_epi64($1))", o.args),

    vstore1_4x32f   := (self, o, i, is) >> Print(Blanks(i), self.prefix("_mm_store_ss",  o.args), ";\n"),
    vstore_2l_4x32f := (self, o, i, is) >> Print(Blanks(i), self.prefix("_mm_storel_pi", o.args), ";\n"),
    vstore_2h_4x32f := (self, o, i, is) >> Print(Blanks(i), self.prefix("_mm_storeh_pi", o.args), ";\n"),
    vstoreu_4x32f   := (self, o, i, is) >> Print(Blanks(i), self.prefix("_mm_storeu_ps", o.args), ";\n"),
    vstoreu2_4x32f  := (self, o, i, is) >> Print(Blanks(i), self.printf("_mm_storel_epi64($1, _mm_castps_si128($2));\n",
        o.args)),

    vstoremsk_4x32f := (self, o, i, is) >> Print(Blanks(i),
	self.printf("_mm_maskmoveu_si128(_mm_castps_si128($2), _mm_set_epi32($3, $4, $5, $6), $1);\n",
            [o.args[1], o.args[2]] :: List(Reversed(o.args[3].v), e->e.v))),

    alignr_4x32f := (self, o, i, is) >> self.printf(
	"_mm_castsi128_ps(_mm_alignr_epi8(_mm_castps_si128($1), _mm_castps_si128($2), $3))", [o.args[1], o.args[2], o.args[3].p]),

    # --------------------------------
    # ISA specific : SSE_8x16i
    #
    vzero_8x16i := (self, o, i, is) >> Print("_mm_setzero_si128()"),

    vunpacklo_8x16i  := (self, o, i, is) >> self.prefix("_mm_unpacklo_epi16", o.args),
    vunpackhi_8x16i  := (self, o, i, is) >> self.prefix("_mm_unpackhi_epi16", o.args),
    vunpacklo2_8x16i := (self, o, i, is) >> self.prefix("_mm_unpacklo_epi32", o.args),
    vunpackhi2_8x16i := (self, o, i, is) >> self.prefix("_mm_unpackhi_epi32", o.args),
    vunpacklo4_8x16i := (self, o, i, is) >> self.prefix("_mm_unpacklo_epi64", o.args),
    vunpackhi4_8x16i := (self, o, i, is) >> self.prefix("_mm_unpackhi_epi64", o.args),

    vpacks_8x16i     := (self, o, i, is) >> self.prefix("_mm_packs_epi16",    o.args),
    vpackus_8x16i    := (self, o, i, is) >> self.prefix("_mm_packus_epi16",   o.args),

    vshuffle2_8x16i := (self, o, i, is) >> self.printf(
	"_mm_castps_si128(_mm_shuffle_ps(_mm_castsi128_ps($1), _mm_castsi128_ps($2), $3))", o.args),

    vshuffle4_8x16i := (self, o, i, is) >> self.printf(
	"_mm_castpd_si128(_mm_shuffle_pd(_mm_castsi128_pd($1), _mm_castsi128_pd($2), $3))", o.args),

    vload1_8x16i := (self, o, i, is) >> self.printf("_mm_insert_epi16($1, $2, $3)", o.args),
    vload2_8x16i := (self, o, i, is) >> self.prefix("_mm_cvtsi32_si128", o.args),
    vload4_8x16i := (self, o, i, is) >> self.prefix("_mm_loadl_epi64", o.args),
    vloadu_8x16i := (self, o, i, is) >> self.prefix("_mm_loadu_si128", o.args),

    vextract1_8x16i := (self, o, i, is) >> self.prefix("_mm_extract_epi16", o.args),
    vextract2_8x16i := (self, o, i, is) >> self.prefix("_mm_cvtsi128_si32", o.args),
    vstoreu_8x16i   := (self, o, i, is) >> Print(Blanks(i), self.prefix("_mm_storeu_si128", o.args), ";\n"),
    vstore4_8x16i   := (self, o, i, is) >> Print(Blanks(i), self.prefix("_mm_storel_epi64", o.args), ";\n"),
    vstoremsk_8x16i := (self, o, i, is) >> Print(Blanks(i),
	self.printf("_mm_maskmoveu_si128($2, _mm_set_epi16($3), $1);\n",
        [o.args[1], o.args[2], () -> PrintCS(Reversed(o.args[3]))])),

    vushuffle2_8x16i  := (self, o, i, is) >> self(o.binop(o.args[1], o.args[1], o.args[2]), i, is),
    vushufflelo_8x16i := (self, o, i, is) >> self.prefix("_mm_shufflelo_epi16", o.args),
    vushufflehi_8x16i := (self, o, i, is) >> self.prefix("_mm_shufflehi_epi16", o.args),

    interleavedmask_8x16i := (self, o, i, is) >> self.printf(
	"_mm_movemask_epi8(_mm_unpacklo_epi8(_mm_packs_epi16($1, _mm_setzero_si128()), _mm_packs_epi16($2, _mm_setzero_si128())))",
	o.args),

    alignr_8x16i := (self, o, i, is) >> self.printf("_mm_alignr_epi8($1, $2, $3)", [o.args[1], o.args[2], o.args[3].p]),

    # FF: NOTE: couldnt figure out how to use the general case with type propagation etc...
    cmplt_8x16i := (self, o, i, is) >> self.printf("_mm_cmplt_epi16($1, $2)", o.args),

    # SSSE3 8x16i instructions
    chs_8x16i := (self, o, i, is) >> self.prefix("_mm_sign_epi16", o.args),
    vushuffle_8x16i := (self, o, i, is) >> self.prefix("_mm_shuffle_epi8", o.args),

    # --------------------------------
    # ISA specific : SSE_16x8i
    #
    vloadu_16x8i  := (self, o, i, is) >> self.prefix("_mm_loadu_si128", o.args),
    vstoreu_16x8i := (self, o, i, is) >> Print(Blanks(i), self.prefix("_mm_storeu_si128", o.args), ";\n"),

    vunpacklo_16x8i  := (self, o, i, is) >> self.prefix("_mm_unpacklo_epi8", o.args),
    vunpackhi_16x8i  := (self, o, i, is) >> self.prefix("_mm_unpackhi_epi8", o.args),
    vunpacklo2_16x8i := (self, o, i, is) >> self.prefix("_mm_unpacklo_epi16", o.args),
    vunpackhi2_16x8i := (self, o, i, is) >> self.prefix("_mm_unpackhi_epi16", o.args),
    vunpacklo4_16x8i := (self, o, i, is) >> self.prefix("_mm_unpacklo_epi32", o.args),
    vunpackhi4_16x8i := (self, o, i, is) >> self.prefix("_mm_unpackhi_epi32", o.args),
    vunpacklo8_16x8i := (self, o, i, is) >> self.prefix("_mm_unpacklo_epi64", o.args),
    vunpackhi8_16x8i := (self, o, i, is) >> self.prefix("_mm_unpackhi_epi64", o.args),

    vushufflelo2_16x8i := (self, o, i, is) >> self.prefix("_mm_shufflelo_epi16", o.args),
    vushufflehi2_16x8i := (self, o, i, is) >> self.prefix("_mm_shufflehi_epi16", o.args),
    vushuffle4_16x8i   := (self, o, i, is) >> self.prefix("_mm_shuffle_epi32", o.args),

    interleavedmasklo_16x8i := (self, o, i, is) >> Print("_mm_movemask_epi8(_mm_unpacklo_epi8(",o.args[1],",",o.args[2],"))"),
    interleavedmaskhi_16x8i := (self, o, i, is) >> Print("_mm_movemask_epi8(_mm_unpackhi_epi8(",o.args[1],",",o.args[2],"))"),
    average_16x8i           := (self, o, i, is) >> Print("_mm_avg_epu8(",o.args[1],",",o.args[2],")"),
    vmovemask_16x8i         := (self, o, i, is) >> self.prefix("_mm_movemask_epi8", o.args),

    # XXX NOTE XXXX
    # Also fix other vstoremsk's. The problem here is that after latest changes to Spiral
    # the last argument (list of strings) in vstoremsg gets wrapped into V, and strings too
    # this is super stupid+ugly
    vstoremsk_16x8i := (self, o, i, is) >> Print(Blanks(i), self.printf("_mm_maskmoveu_si128($2, _mm_set_epi8($3), $1);\n",
        [o.args[1], o.args[2], () -> PrintCS(Reversed(List(_unwrapV(o.args[3]), _unwrapV)))])),

    addsub_4x32f := (self, o, i, is) >> Checked(Length(o.args) = 2,
        CondPat(o,
           [addsub_4x32f, @TReal, @TVect], self(addsub_4x32f(vdup(o.args[1],o.t.size), o.args[2]), i, is),
           [addsub_4x32f, @TVect, @TReal], self(addsub_4x32f(o.args[1], vdup(o.args[2],o.t.size)), i, is),
           [addsub_4x32f, @TInt,  @TVect], self(addsub_4x32f(vdup(_toReal(o.args[1]),o.t.size), o.args[2]), i, is),
           [addsub_4x32f, @TVect, @TInt],  self(addsub_4x32f(o.args[1], vdup(_toReal(o.args[2]),o.t.size)), i, is),
           [addsub_4x32f, @TVect, @TVect], self.printf("_mm_addsub_ps($1, $2)", [o.args[1], o.args[2]]),
           Error("Don't know how to unparse <o>. Unrecognized type combination")
    )),

    hadd_4x32f   := (self, o, i, is) >> self.printf("_mm_hadd_ps($1, $2)", [o.args[1], o.args[2]]),

    vloadu_16x8i := (self, o, i, is) >>  self.prefix("_mm_loadu_si128", o.args),

    # --------------------------------
    # ISA specific : SSE_4x32i
    #
    vunpacklo_4x32i := (self, o, i, is) >> self.prefix("_mm_unpacklo_epi32", o.args),
    vunpackhi_4x32i := (self, o, i, is) >> self.prefix("_mm_unpackhi_epi32", o.args),
    vpacks_4x32i    := (self, o, i, is) >> self.prefix("_mm_packs_epi32", o.args),
    # 32 bit integer shuffles are *not* the same as 32 bit float, but similar to 16 bit integer shuffles
    vushuffle_4x32i := (self, o, i, is) >> self.prefix("_mm_shuffle_epi32", o.args),
    vshuffle_4x32i  := (self, o, i, is) >> self.printf(
	"_mm_castps_si128(_mm_shuffle_ps(_mm_castsi128_ps($1), _mm_castsi128_ps($2), $3))", o.args),

    # subvector unparsing not yet done...
    vload1_4x32i := (self, o, i, is) >> self.prefix("_mm_cvtsi32_si128", o.args), #svpcprint guy
    vload2_4x32i := (self, o, i, is) >> self.prefix("_mm_loadl_epi64", o.args),
    vload2_4x32i := (self, o, i, is) >> self.prefix("_mm_loadl_epi64", o.args),
    vloadu_4x32i := (self, o, i, is) >> self.prefix("_mm_loadu_si128", o.args),

    vextract_4x32i  := (self, o, i, is) >> self.prefix("_mm_cvtsi128_si32", o.args),
    vstoreu_4x32i   := (self, o, i, is) >> Print(Blanks(i), self.prefix("_mm_storeu_si128", o.args), ";\n"),
    vstore2_4x32i   := (self, o, i, is) >> Print(Blanks(i), self.prefix("_mm_storel_epi64", o.args), ";\n"),
    vstoremsk_4x32i := (self, o, i, is) >> Print(Blanks(i),
	self.printf("_mm_maskmoveu_si128($2, _mm_set_epi16($3), $1);\n", [o.args[1], o.args[2], ()->PrintCS(Reversed(o.args[3]))])),
        # complicated, buggy

    interleavedmask_4x32i := (self, o, i, is) >> self.printf(
	"_mm_movemask_epi8(_mm_packs_epi16(_mm_unpacklo_epi16(_mm_packs_epi16($1, $3), _mm_packs_epi16($2, $3)), $3))",
	o.args :: ["_mm_setzero_si128()"]),

    vcvt_4x32_i2f  := (self, o, i, is) >> self.prefix("_mm_cvtepi32_ps", o.args),
    vcvt_4x32_f2i  := (self, o, i, is) >> self.prefix("_mm_cvtps_epi32", o.args),
    vcvtt_4x32_f2i := (self, o, i, is) >> self.prefix("_mm_cvttps_epi32", o.args),

    testz_4x32i := (self, o, i, is) >> self.prefix("_mm_testz_si128", o.args),
    testc_4x32i := (self, o, i, is) >> self.prefix("_mm_testc_si128", o.args),
    testnzc_4x32i := (self, o, i, is) >> self.prefix("_mm_testnzc_si128", o.args),

    # --------------------------------
    # ISA specific : SSE_2x64i
    #
    vunpacklo_2x64i := (self, o, i, is) >> self.prefix("_mm_unpacklo_epi64", o.args),
    vunpackhi_2x64i := (self, o, i, is) >> self.prefix("_mm_unpackhi_epi64", o.args),
    vshuffle_2x64i  := (self, o, i, is) >> self.printf(
	"_mm_castpd_si128(_mm_shuffle_pd(_mm_castsi128_pd($1), _mm_castsi128_pd($2), $3))", o.args),
    vushuffle_2x64i := (self, o, i, is) >> self(o.binop(o.args[1], o.args[1], o.args[2]), i, is),

    # --------------------------------
    # tcast __m128 <-> __m128i

    tcast := (self, o, i, is) >> let(
        isa := _isa(self),
        i128 := @.cond(x-> let( t := When(IsType(x), x, x.t), IsVecT(t) and self.ctype(t, isa)="__m128i")),
        f128 := @.cond(x-> let( t := When(IsType(x), x, x.t), IsVecT(t) and self.ctype(t, isa)="__m128" )),
        d128 := @.cond(x-> let( t := When(IsType(x), x, x.t), IsVecT(t) and self.ctype(t, isa)="__m128d" )),
        CondPat(o,
            [tcast, i128, f128], self.prefix("_mm_castps_si128", [o.args[2]]),
            [tcast, i128, d128], self.prefix("_mm_castpd_si128", [o.args[2]]),
            [tcast, f128, i128], self.prefix("_mm_castsi128_ps", [o.args[2]]),
            [tcast, i128, i128], self(o.args[2], i, is),
            [tcast, f128, f128], self(o.args[2], i, is),
            Inherited(o, i, is))),

    tcvt := (self, o, i, is) >> self.printf("(($1)($2))", [o.args[1], o.args[2]]),

    #NOTE: finish this, it should look at TVect.size and instruction set for figuring out exactly what to do
    vcastizxlo := (self, o, i, is) >> self(vunpacklo_16x8i(o.args[1], o.t.zero()), i, is),
    vcastizxhi := (self, o, i, is) >> self(vunpackhi_16x8i(o.args[1], o.t.zero()), i, is),
    vcastuzxlo := ~.vcastizxlo,
    vcastuzxhi := ~.vcastizxhi,

    average := (self, o, i, is) >> CondPat(o,
        [average, @TVect, @TVect], let(
            sfx := self.ctype_suffix(o.t, _isa(self)),
            Cond( sfx in ["epu8", "epu16"],
                self.printf("_mm_avg_$1($2, $3)",[sfx, o.args[1],o.args[2]]),
                Error("finish SSE unparser"))),
        Inherited(o, i, is)),
));


# Copyright (c) 2018-2021, Carnegie Mellon University
# See LICENSE for details

#Class(MultiBufBB, RuleSet);
#RewriteRules(MultiBufBB, rec(
#     remove_bbs := Rule(@(1, Compose, e->ForAll(e.children(), c->ObjId(c)=DistSum)),
#        e -> ComposeDists(@(1).val.children()) )
#));

applyCellInplace := function(sums, opts)
    SubstBottomUp(sums, GathDist, e->Inplace(e));
    SubstBottomUp(sums, GathRecv, e->Inplace(e));
    SubstBottomUp(sums, ScatDist, e->Inplace(e));

    SubstBottomUp(sums, ScatMem, e->Inplace(e));
    SubstBottomUp(sums, GathMem, e->Inplace(e));

    # BB(Inplace()) -> Inplace() (So that a copy operation need not take place)
    SubstBottomUp(sums, [BB, Inplace], e->e.rChildren()[1]);
    return(sums);
end;

#s := SubstBottomUp(s, ScatMem, e->Inplace(e));
#s := SubstBottomUp(s, GathMem, e->Inplace(e));
#s := SubstBottomUp(s, [BB, Inplace], e->e.rChildren()[1]);

# DistSum(Scat__ * MultiBufISum() * Gath__) -> DistSumLoop(MultiBufISum(ScatMem * () * GathMem))

# DistSum has to be converted to a DistSumLoop because DistSum expects its Scat
# and Gath to span the entire expanse of the computation, and therefore assumes
# an appropriate Scat/Gath. DistSumLoop does not.

doMultiBufBC := function(sums, opts)
    Error("doMultiBufBC: BP");
    return(sums);
end;

#F Dist(MultiBuf) is for independent DFTs running on separate SPEs (or sets of
#F SPEs), each multibuffered separately
Class(DistMultiBuf, RuleSet);

RewriteRules(DistMultiBuf, rec(
     # HACK: rule assumes we do only independent DFTs
     join_dist_multibuf_compose := Rule([@(1,DistSum), @, @(2,Compose, e->Length(Collect(e.rChildren()[1], MultiBufISum))=1) ],
       e -> let(
          dsum := @(1).val,
          Error(""),
          msum := Collect(@(2).val, MultiBufISum)[1],
          sp   := Collect(dsum, @(3,[ScatDist, ScatSend]))[1],
          sm   := msum.scatmem,
          sdm  := fCompose(sp.func, fTensor(sm.func, fId(sm.pkSize/sp.pkSize))),

          gp   := Collect(dsum, @(3,[GathDist, GathRecv]))[1],
          gm   := msum.gathmem,
          gdm  := fCompose(gp.func, fTensor(gm.func, fId(gm.pkSize/gp.pkSize))),

          DistSumLoop(dsum.P, dsum.var, dsum.domain,
            MultiBufISum(msum.var, msum.domain,
              #ScatDirectMem(sdm, sp.pkSize, dsum.P, dsum.var),
              ScatMem(sdm, sp.pkSize),
              msum.rChildren()[2],
              #GathDirectMem(gdm, gp.pkSize, dsum.P, dsum.var)
              GathMem(gdm, gp.pkSize)
            )
          )
       )
     ),

     join_dist_multibuf_separate := Rule(@(1, DistSum, e->Length(Collect(e.rChildren()[2], MultiBufISum))=1),
       e -> let(
          dsum := @(1).val,
          msum := Collect(@(1).val, MultiBufISum)[1],

          sp   := Collect(dsum, @(3,[ScatDist, ScatSend]))[1],
          sm   := msum.scatmem,
          spm  := fCompose(fTensor(sp.func, fId(sp.pkSize/sm.pkSize)), sm.func),

          gp   := Collect(dsum, @(3,[GathDist, GathRecv]))[1],
          gm   := msum.gathmem,
          gpm  := fCompose(fTensor(gp.func, fId(gp.pkSize/gm.pkSize)), gm.func),

          DistSumLoop(dsum.P, dsum.var, dsum.domain,
            MultiBufISum(msum.var, msum.domain,
              #ScatDirectMem(spm, sp.pkSize, dsum.P, dsum.var),
              ScatMem(spm, sm.pkSize),
              msum.rChildren()[2],
              #GathDirectMem(gpm, gp.pkSize, dsum.P, dsum.var)
              GathMem(gpm, gm.pkSize)
            )
          )
       )
     )

));

#F MultiBuf(Dist) is for parallel DFTs that are multibuffered
#F We pull out a parallel scat (gath) function into the multibufisum's scat (gath) function
#F Sums becomes invalid (strictly speaking) because we now use loop vars outside of their definition.
Class(MultiBufDist, RuleSet);
RewriteRules(MultiBufDist, rec(
    # This has to be done exactly once, else will loop infinitely. Look at hack below
    #NOTE: The rule below doesn't seem to fire. Probably doesn't match becacuse of the # of @'s at the end in this line:
     join_multibuf_dist_compose := Rule([@(1,[MultiBufISum,MemISum]), @(2,[Compose,ComposeDists], e->Length(Collect(e.rChildren()[2], DistSum))=1), @, @ ],
       e -> let(

          #HACK: clean way of getting the right gath/scat is to ask for the Compose's leftmost Scat/Gath
          msum := @(1).val,
          dsums := Collect(@(2).val, DistSum),
          dsum1 := dsums[1],
          dsum2 := dsums[Length(dsums)],

          sd   := Collect(dsum1, @(3,[ScatDist, ScatSend]))[1],
          sm   := msum.scatmem,
          sdm  := fCompose(fTensor(sm.func, fId(sm.pkSize/sd.pkSize)), sd.func),

          gd   := Collect(dsum2, @(3,[GathDist, GathRecv]))[1],
          gm   := msum.gathmem,
          gdm  := fCompose(fTensor(gm.func, fId(gm.pkSize/gd.pkSize)), gd.func),

          isum := When(ObjId(@(1).val)=MultiBufISum, MultiBufISumFinal, MemISumFinal),

          # HACK: Converting to a MultiBufISumFinal so this rule won't match more than once
          isum(msum.var, msum.domain,
              ScatMem(sdm, sd.pkSize),
              msum.rChildren()[2],    #NOTE: is this okay?
              GathMem(gdm, gd.pkSize)
          )
       )
     ),

     join_multibuf_dist_separate := Rule(@(1, MultiBufISum, e->Length(Collect(e.rChildren()[2], DistSum))=1),
       e -> let(

          msum := @(1).val,
          dsum := Collect(@(1).val, DistSum)[1],

          sd   := Collect(dsum, @(3,[ScatDist, ScatSend]))[1],
          sm   := msum.scatmem,
          sdm  := fCompose(fTensor(sm.func, fId(sm.pkSize/sd.pkSize)), sd.func),

          gd   := Collect(dsum, @(3,[GathDist, GathRecv]))[1],
          gm   := msum.gathmem,
          gdm  := fCompose(fTensor(gm.func, fId(gm.pkSize/gd.pkSize)), gd.func),

          # HACK: Converting to a MultiBufISumFinal so this rule won't match more than once
          MultiBufISumFinal(msum.var, msum.domain,
              ScatMem(sdm, sd.pkSize),
              msum.rChildren()[2],    #NOTE: is this okay?
              GathMem(gdm, gd.pkSize)
          )
       )
     )
));

#F MultiBufDist_large is for large DFTs that must be multibuffered in parts.
#F Since the multibuf loop is the outer loop, it assumes the "chip" can bring
#F data from memory on to it. In reality, "chips" don't exist -- only cores do. So
#F this rule distributed chip access across the core. It's different from the
#F MultiBufDist rule in that it doesn't interact with the inside DistSum's
#F Scatters or gathers. It does use the parallel loop var outside of its
#F definition when done.

Class(MultiBufDist_large, RuleSet);

RewriteRules(MultiBufDist_large, rec(
    distribute_scatgathmems := Rule(@(1, MultiBufISum, e->Length(Collect(e.rChildren()[2], DistSum))>=1 ),
      e -> let(

        msum  := @(1).val,
        dsum  := Collect(@(1).val, DistSum)[1],
        sm    := msum.scatmem,
        gm    := msum.gathmem,

        sf    := When(sm.func.domain()=1, dsum.P, 1),
        gf    := When(gm.func.domain()=1, dsum.P, 1),

        sfunc := When(sf=1, sm.func, fTensor(sm.func, fId(sf))),
        gfunc := When(gf=1, gm.func, fTensor(gm.func, fId(gf))),

        Ns    := sf*sm.func.range()/msum.domain,
        ns    := sf*sm.func.domain()/dsum.P,
        bs    := (dsum.var * ns),

        Ng    := gf*gm.func.range()/msum.domain,
        ng    := gf*gm.func.domain()/dsum.P,
        bg    := (dsum.var * ng),

        smnew := ScatMem(fCompose(sfunc, H(Ns, ns, bs, 1)), sm.pkSize/sf),
        gmnew := GathMem(fCompose(gfunc, H(Ng, ng, bg, 1)), gm.pkSize/gf),

        #Error("BP"),

        isum := When(ObjId(@(1).val)=MultiBufISum, MultiBufISumFinal, MemISumFinal),

        isum(msum.var, msum.domain,
            smnew,
            msum.rChildren()[2],    #NOTE: is this okay?
            gmnew
        )
      )
    )
));

# HACK!!!
# This is the way the system works: the above rules (RulesVRC) passes VRCL and
# VRCR tags to the scat and gath of a multibufisum. BUT, these have
# cannotchangedataformat set to true, AND, they're not exposed to the VRC
# rules. So the VRC rules don't really touch these. So in some cases, they end
# up with VRCL and VRCR, when in reality, they don't have either. The following
# rule "fixes" this problem by simply assuming they're either VRCs or VRCLRs.
# Hackity hack.
RewriteRules(RulesVRC, rec(
    VRC_MultiBufISum := Rule([@(1, [VRC,VRCL,VRCR,VRCLR]), @(2, [MultiBufISum])],
       e->let(v := @(1).val,
              m := @(2).val,
              MultiBufISum(m.var, m.domain,
                ObjId(v)(m.scatmem, v.v),
                ObjId(v)(m._children[1], v.v),
                ObjId(v)(m.gathmem, v.v))
              )
       ),
    VRC_MemISum := Rule([@(1, [VRC,VRCL,VRCR,VRCLR]), @(2, [MemISum])],
       e->let(v := @(1).val,
              m := @(2).val,
              MemISum(m.var, m.domain,
                ObjId(v)(m.scatmem, v.v),
                ObjId(v)(m._children[1], v.v),
                ObjId(v)(m.gathmem, v.v))
              )
       )
));

RewriteRules(CellVRCTerm, rec(
    VRC_ScatMem_Term := Rule([@(1, [RC, VRCL, VRCR, VRC,VRCLR]), @(2, [ScatMem])],
    e->ScatMem(@(2).val.func, @(2).val.pkSize*2)),

    VRC_GathMem_Term := Rule([@(1, [RC, VRCL, VRCR, VRC,VRCLR]), @(2, [GathMem])],
    e->GathMem(@(2).val.func, @(2).val.pkSize*2)),
));

RewriteRules(RulesRC, rec(
    RC_MultiBufISum := Rule([RC, @(1, MultiBufISum)],
        e -> let(s:=@(1).val, MultiBufISum( s.var, s.domain, s.scatmem, RC(s.child(1)), s.gathmem ))
    ),
));

# To convert Compose(A, B) -> ComposeStreams(A, B) when A,B=MultiBufISum
# ObjId(c)=DistSumLoop below is a hack. Obviously, DistSumLoop does not necessarily imply composing streams.
Class(RulesComposeStreams, RuleSet);
RewriteRules(RulesComposeStreams, rec(
    stream_compose := Rule(@(1, Compose, e->ForAll(e.children(), c->(ObjId(c)=MultiBufISum  or ObjId(c)=MultiBufISumFinal or ObjId(c)=DistSumLoop) )),
        e -> ComposeStreams(@(1).val.children()) )
));

# Pull Diag into MultiBufISum. D*MBufISum(SAG) -> MBufISum(SDAG) and
# MBufISum(SAG)*D -> MBufISum(SADG) Unlike the DistSum rules to do the same
# thing, this combining has to be done in a single step because the Scat and
# Gath of the MultiBufISum are not exposed.

RewriteRules(RulesDiagStandalone, rec(
 #  MBufISum(SAG)*D
 CellPullInCommuteGathDiag := ARule(Compose, [ @(1, MultiBufISum), @(2, [Prm, Gath, Diag, RCDiag]) ],
    e->let(msum := @(1).val,
           diag := @(2).val,
           gath := msum.gathmem,
           newdiag := Diag(fCompose(diag.element, fTensor(gath.func, fId(gath.pkSize)))).attrs(diag),
        [ MultiBufISum(msum.var, msum.domain, msum.scatmem, msum.child(1) * newdiag, msum.gathmem) ]
       )
 ),

 CellPullInCommuteScatDiag := ARule(Compose,  [ @(1, [RCDiag, Diag, Prm, Scat]), @(2, MultiBufISum) ],
    e->let(msum := @(2).val,
           diag := @(1).val,
           scat := msum.scatmem,
           newdiag := Diag(fCompose(diag.element, fTensor(scat.func, fId(scat.pkSize))  )).attrs(diag),
        [ MultiBufISum(msum.var, msum.domain, msum.scatmem, newdiag * msums.child(1), msum.gathmem) ]
       )
 ),

 # NOTE
 ## Gath * RCDiag
 #CellCommuteGathRCDiag := ARule( Compose,
 #      [ [@(1, [ GathDist, GathRecv ]), [@(0,fTensor), ..., [fId,@(2).cond(IsEvenInt)]]],
 #     @(4, RCDiag) ],
 # e -> [ RCDiag(fCompose(@(4).val.element, @(0).val), @(4).val.post),
 #        @(1).val ]),

 ## RCDiag * Scat
 #CellCommuteRCDiagScat := ARule( Compose,
 #      [ @(4, RCDiag),
 #    [@(1, [Scat, ScatDist, ScatSend]), [@(0,fTensor), ..., [fId,@(2).cond(IsEvenInt)]]] ],
 # e -> [ @(1).val,
 #        RCDiag(fCompose(@(4).val.element, @(0).val), @(4).val.post) ]),

));


# Copyright (c) 2018-2021, Carnegie Mellon University
# See LICENSE for details

Rows := s -> Cond(
    IsMat(s),
        Length(s),

    IsBound(s.rng) and IsBound(s.dmn),
        s.dims()[1],

    IsValue(s) or IsSymbolic(s),
        Checked(IsArrayT(s.t), IsArrayT(s.t.t),
	    s.t.size),

    s.dimensions[1]
);

Cols := s -> Cond(
    IsMat(s),
        Length(s[1]),

    IsBound(s.rng) and IsBound(s.dmn),
        s.dims()[2],

    IsValue(s) or IsSymbolic(s),
        Checked(IsArrayT(s.t), IsArrayT(s.t.t),
	    s.t.t.size),

    s.dimensions[2]
);

olRows := s -> Flat([Rows(s)]);
olCols := s -> Flat([Cols(s)]);

#F SPL(<rec>)
#F    Set operations field to SPLOps. This function should be called
#F    to initialize SPL instances.
SPL := function(record)
   record.operations := SPLOps;
   return record;
end;

HashAsSPL := o -> Cond(
    IsList(o),
        List(o, HashAsSPL),
    not IsRec(o) or (not IsBound(o.hashAs) and not IsBound(o.from_rChildren)),
        o,
    IsBound(o.hashAs),
        o.hashAs(),
    IsValue(o),
        o.v,
    o.from_rChildren(List(o.rChildren(), HashAsSPL))
);

# ==========================================================================
# ClassSPL
#
# Base class for all SPL constructs
# ==========================================================================
Class(ClassSPL, AttrMixin, rec(
    isSPL := true,
    transposed := false,

#DD this allows objects which are not TaggedNonTerminals to drop tags automagically
#DD and without error.
    withTags := (self, t) >> self,

    _short_print := false,
    _newline := i ->  Print("\n", Blanks(i)),
    _indent  := i ->  Print(Blanks(i)),
    _indentStr := Blanks,

    __call__ := meth(arg)
        local self, params, nump, A,p,res,h,lkup;
        self := arg[1];
        params := arg{[2..Length(arg)]};
        nump := Length(params);

        if not IsBound(self.new)  then
            Error("Constructor for this class is not implemented");
        elif IsBound(self.abbrevs) and self.abbrevs <> [] then
            for A in self.abbrevs do
                if NumArgs(A) = -1 or NumArgs(A) = nump then
                    params := ApplyFunc(A, params);
                fi;
            od;
        fi;

        if NumArgs(self.new)-1 <> Length(params) then
            Error("Constructor requires ", NumArgs(self.new)-1, " parameters (",
                Length(arg)-1, " given): ",
                ParamsMeth(self.new));
        else
            h := self.hash;
            if h<>false then
                lkup := h.objLookup(self, params);
                if lkup[1] <> false then return lkup[1]; fi;
            fi;

            res := ApplyFunc(self.new, params);

            if h<>false then return h.objAdd(res, lkup[2]);
            else return res;
            fi;
        fi;
    end,

    hash := false,

    checkDims := self >> DimensionsMat(MatSPL(self)) = self.dimensions,

    #-----------------------------------------------------------------------
    # create a new object with .<name> field set to true
    setAttr := meth(self, name)
        local s;
        s:= Copy(self);
        s.(name) := true;
        return s;
    end,
    # ----------------------------------------------------------------------
    # create a new object with .<name> field set to <val>
    setAttrTo := meth(self, name, val)
        local s;
        s:= Copy(self);
        s.(name) := val;
        return s;
    end,

    #---------Backwards Compatibility for the new dimension system------
    setDims := meth(self) self.dimensions := self.dims(); return self; end,

    dims := self >> [ StripList(List(self.rng(), l -> l.size)),
	              StripList(List(self.dmn(), l -> l.size)) ],

    advdims := (self) >> let(d := self.dims(), [ [[ d[1] ]], [[ d[2] ]] ]),
    arity   := (self) >> List(self.dims(), e -> Length(Flat([e]))),

    TType:=TUnknown,

    rng := meth(self) local d;
        if IsBound(self.dims) then
            d := Flat([self.dims()[1]]);
        else
            d := [self.dimensions[1]];
        fi;
        if IsBound(self.a.t_out) then
            return List(TransposedMat([self.a.t_out, d]), e -> TArray(e[1], e[2]));
        else
            return List(d, e -> TArray(self.TType, e));
        fi;
    end,

    dmn := meth(self) local d;
        if IsBound(self.dims) then
            d := Flat([self.dims()[2]]);
        else
            d := [self.dimensions[2]];
        fi;
        if IsBound(self.a.t_in) then
            return List(TransposedMat([self.a.t_in, d]), e -> TArray(e[1], e[2]));
        else
            return List(d, e -> TArray(self.TType, e));
        fi;
    end,

    free := self >> Union(List(self.rChildren(), FreeVars)),

    equals := (self, o) >>
        ObjId(self) = ObjId(o) and self.rChildren() = o.rChildren() and self.a = o.a,

    lessThan := (self, o) >> Cond(
        ObjId(self) <> ObjId(o), ObjId(self) < ObjId(o),
        [ ObjId(self), self.rChildren(), self.a ] < [ ObjId(o), o.rChildren(), o.a ]
    ),

    from_rChildren := (self, rch) >> ApplyFunc(ObjId(self), rch).appendAobj(self),

    print := (self, i, is) >> self._print(self.rChildren(), i, is),

    _print := meth(self, ch, indent, indentStep)
        local s, first, newline;

	if self._short_print or ForAll(ch, x->not IsRec(x) or IsSPLSym(x) or IsSPLMat(x)) then
	    newline := Ignore;
	else
	    newline := self._newline;
	fi;

	first := true;
        Print(self.__name__, "(");
	for s in ch do
            if(first) then first:=false;
            else Print(", "); fi;
            newline(indent + indentStep);
            When(IsSPL(s) or (IsRec(s) and IsBound(s.print) and NumGenArgs(s.print)=2),
                 s.print(indent + indentStep, indentStep),
		 Print(s));
	od;
	newline(indent);
	Print(")");
        self.printA();

	if IsBound(self._setDims) then
            Print(".overrideDims(", self._setDims, ")");
	fi;
    end,

    printlatex := meth(self)
        local s, first, newline, i;

	first := true;
        Print("(");
        i := Length(self.rChildren());
        for s in self.rChildren() do
            When(IsSPL(s) or (IsRec(s) and IsBound(s.printlatex) and NumGenArgs(s.printlatex)=0),
                 s.printlatex(),
		 Print(s));
                 if i >=2 then
                    Print(" ", When(IsBound(self.latexSymbol), self.latexSymbol, ""), " ");
                 fi;
                 i := i-1;
	od;
	Print(")");
    end,

    overrideDims := (self, dims) >> CopyFields(self, rec(_setDims := dims, dimensions := dims)),

    terminate     := self >> self.from_rChildren(List(self.rChildren(), x->When(IsSPL(x), x.terminate(), x))),

    # ------------------------- Required methods ---------------------------\
    transposeSymmetric := True,
    dims          := meth(self) Error("Not implemented"); end,
    isPermutation := meth(self) Error("Not implemented"); end,
    isTerminal    := meth(self) Error("Not implemented"); end,
    isReal        := meth(self) Error("Not implemented"); end,
    isInplace     := self >> Rows(self)=Cols(self) and let(ch:=self.children(), Cond(Length(ch)=0, false, ForAll(ch, x->x.isInplace()))),
    children      := meth(self) return []; end,
    numChildren   := meth(self) return 0; end,
    child         := meth(self,n) Error("Not implemented"); end,
    setChild      := meth(self,n,what) Error("Not implemented"); end,
    toAMat        := meth(self) Error("Not implemented"); end,
    transpose     := meth(self) Error("Not implemented"); end,
    conjTranspose  := meth(self) Error("Not implemented"); end,
));

#F <SPL> * <SPL>
#F <scalar> * <SPL>
#F   is equivalent to ComposeSPL and ScalarMultiple resp.
#F
SPLOps.\* := (S1, S2) ->
    Cond(IsSPL(S1) and IsSPL(S2),   Compose(S1, S2),
         IsSPL(S2),                 Scale(S1, S2),
     IsSPL(S1),                 Scale(S2, S1),
     Error("do not know how to compute <S1> * <S2>"));

#F <SPL> + <SPL>
#F   is equivalent to SUM(<S1>, <S2>)
#F
SPLOps.\+ := (S1, S2) ->
    Cond(IsSPL(S1) and IsSPL(S2),   SUM(S1, S2),
     Error("do not know how to compute <S1> + <S2>"));

#F S1 ^ S2
#F   is equivalent to ConjugateSPL(S1, S2).
#F
SPLOps.\^ := (S1, S2) ->
    When(IsSPL(S1) and IsSPL(S2),
         Conjugate(S1, S2),
     Error("do not know how to compute S1 ^ S2"));