## anova between cell types for DMP analysis
## sets double negative as baseline
## bumphunter for regional analysis
## use Zhou et al hg38 probe annotation to align with other omics data.

anovaModel<-function(row, celltype, age,sex){
	model<-lm(row ~ celltype + age + sex)
	null<-lm(row ~ age + sex)
	tmp<-summary(model)$coefficients
	regCoeffs<-c(t(tmp[c("celltypeNeuN +ve", "celltypeSox10 +ve", "celltypeIRF8 +ve"), c(1,4)]), anova(model,null)[2,6])
	return(regCoeffs)
}

library(ChIPseeker)
library(Gviz)
library(TxDb.Hsapiens.UCSC.hg38.knownGene)
library(doParallel)
library(data.table)

### test cell comp differences
source("rmdConfig.mrc")
setwd(dataDir)
load(normData)

## remove rs probes
celltypenormbeta<-celltypenormbeta[grep("rs", rownames(celltypenormbeta), invert = TRUE),]

## calc cell type means including Total
cellMeans<-NULL
for(each in sort(unique(pheno$Cell.type))){
	cellMeans<-cbind(cellMeans, apply(celltypenormbeta[,which(pheno$Cell.type == each)], 1, mean))
}
colnames(cellMeans)<-sort(unique(pheno$Cell.type))

## calc cell type SDs including Total
cellSDs<-NULL
for(each in sort(unique(pheno$Cell.type))){
	cellSDs<-cbind(cellSDs, apply(celltypenormbeta[,which(pheno$Cell.type == each)], 1, sd))
}
colnames(cellSDs)<-sort(unique(pheno$Cell.type))

## filter to just cell fractions
pheno<-pheno[pheno$Cell.type %in% c("Double -ve","NeuN +ve","Sox10 +ve", "IRF8 +ve"),]
celltypenormbeta<-celltypenormbeta[,pheno$Basename]
pheno$Cell.type<-factor(pheno$Cell.type)

## anova for cell type differences at individual positions
ctOut<-apply(celltypenormbeta, 1, anovaModel, pheno$Cell.type, pheno$Age, pheno$Sex)
ctOut<-t(ctOut)
colnames(ctOut)<-c("NeuN:Coeff", "NeuN:P", "Sox10:Coeff", "Sox10:P",  "IRF8:Coeff", "IRF8:P", "ANOVA:P")

annoObj <-  minfi::getAnnotationObject("IlluminaHumanMethylationEPICanno.ilm10b4.hg19")
all <- minfi:::.availableAnnotation(annoObj)$defaults
newfData <- do.call(cbind, lapply(all, function(wh) {
        minfi:::.annoGet(wh, envir = annoObj@data)
}))

newfData<-newfData[rownames(celltypenormbeta),]

## filter out sites with missing location info
ctOut<-ctOut[!is.na(newfData$pos),]
celltypenormbeta<-celltypenormbeta[rownames(ctOut),]
newfData <- newfData[rownames(ctOut), ]

## looking at ucsc the middle of start,end gives the C position
probeAnno<-fread("../References/EPICArray/EPIC.anno.GRCh38.tsv", sep = "\t", header = TRUE)
probeAnno<-probeAnno[match(rownames(celltypenormbeta), probeAnno$probeID),]
probeAnno<-as.data.frame(probeAnno)

ctOut<-cbind(ctOut, probeAnno[,c("chrm","start", "end", "GeneNames", "GeneClasses", "TranscriptIDs")])
ctOut<-ctOut[which(ctOut$chrm != "*"),]
save(ctOut, file="Analysis/CellType/ANOVABrainCellTypes.rdata")

length(which(ctOut[,7] < 9e-8))
length(which(ctOut[,2] < 9e-8))
length(which(ctOut[,4] < 9e-8))
length(which(ctOut[,6] < 9e-8))

pdf("Analysis/CellType/QQplot.pdf")
par(mfrow = c(2,2))
qq(ctOut[,7], main = "Any")
qq(ctOut[,2], main = "NeuNvsDoubleNeg")
qq(ctOut[,4], main = "Sox10vsDoubleNeg")
qq(ctOut[,6], main = "IRF8vsDoubleNeg")
dev.off()

index<-which(ctOut[,7] < 9e-8)

## look at direction of effect
pdf("Analysis/CellType/HistDMPsMeanDifferences.pdf", width = 12, height = 4)
par(mfrow = c(1,3))
hist(ctOut[which(ctOut[,2] < 9e-8),1], xlab = "Mean Difference", main = "NeuNvsDoubleNegative", ylab = "nDMPs")
signTab<-table(sign(ctOut[which(ctOut[,2] < 9e-8),1]))
signTest<-binom.test(signTab[1], sum(signTab))
text(x =  par("usr")[2],y =  par("usr")[4], pos = 2, offset = -1, paste(signif(signTab[2]/sum(signTab)*100,3), "% DMPs hypermethylated.\nP =", signif(signTest$p.value, 3)), xpd= TRUE)
hist(ctOut[which(ctOut[,4] < 9e-8),3], xlab = "Mean Difference", main = "Sox10vsDoubleNegative", ylab = "nDMPs")
signTab<-table(sign(ctOut[which(ctOut[,4] < 9e-8),3]))
signTest<-binom.test(signTab[1], sum(signTab))
text(x =  par("usr")[2],y =  par("usr")[4], pos = 2, offset = -1, paste(signif(signTab[2]/sum(signTab)*100,3), "% DMPs hypermethylated.\nP =", signif(signTest$p.value, 3)), xpd= TRUE)
hist(ctOut[which(ctOut[,6] < 9e-8),5], xlab = "Mean Difference", main = "IRF8vsDoubleNegative", ylab = "nDMPs")
signTab<-table(sign(ctOut[which(ctOut[,6] < 9e-8),5]))
signTest<-binom.test(signTab[1], sum(signTab))
text(x =  par("usr")[2],y =  par("usr")[4], pos = 2, offset = -1, paste(signif(signTab[2]/sum(signTab)*100,3), "% DMPs hypermethylated.\nP =", signif(signTest$p.value, 3)), xpd= TRUE)
dev.off()

library(VennDiagram)
library(minfi)

# Chart
venn.diagram(
  x = list(rownames(ctOut)[which(ctOut[,2] < 9e-8)], rownames(ctOut)[which(ctOut[,4] < 9e-8)], rownames(ctOut)[which(ctOut[,6] < 9e-8)]),
  category.names = c("NeuNvsDoubleNeg" , "Sox10vsDoubleNeg" , "IRF8vsDoubleNeg"),
  filename = 'Analysis/CellType/VennDiagramCelltypeDMPs.png',
  output=TRUE
)

## for probes that are DMPs in two cell types, same direction and magnitude?

Neun.ES<-cut(ctOut[,1], seq(-1,1,0.05))
Sox10.ES<-cut(ctOut[,3], seq(-1,1,0.05))
Irf8.ES<-cut(ctOut[,5], seq(-1,1,0.05))

pdf("Analysis/CellType/HeatmapSharedDMPsBetweenCellTypes.pdf")
index<-which(ctOut[,2] < 9e-8 & ctOut[,4] < 9e-8)
heatmap(table(Neun.ES[index], Sox10.ES[index]), Rowv = NA, Colv = NA, xlab = "NeuN+ve vs Double-ve", ylab = "Sox10+ve vs Double-ve")
index<-which(ctOut[,2] < 9e-8 & ctOut[,6] < 9e-8)
heatmap(table(Neun.ES[index], Irf8.ES[index]), Rowv = NA, Colv = NA, xlab = "NeuN+ve vs Double-ve", ylab = "Irf8+ve vs Double-ve")
index<-which(ctOut[,4] < 9e-8 & ctOut[,6] < 9e-8)
heatmap(table(Sox10.ES[index], Irf8.ES[index]), Rowv = NA, Colv = NA, xlab = "Sox10+ve vs Double-ve", ylab = "Irf8+ve vs Double-ve")
dev.off()

## how many loci?
dat<-data.frame("ids" = rownames(ctOut), "pval" = ctOut[,"NeuN:P"], "chr" = newfData$chr, "pos" = newfData$pos)
clump.neun<-clump(dat, thres1 = 9e-28, thres2 = 9e-28)

## how many genes?

## bumphunter to call regions

## filter samples with missing Age
celltypenormbeta<-celltypenormbeta[,!is.na(pheno$Age)]
pheno<-pheno[!is.na(pheno$Age),]
## sort
ctOut<-ctOut[order(ctOut$chrm, ctOut$start),]
celltypenormbeta<-celltypenormbeta[rownames(ctOut),]

cl<-clusterMaker(ctOut$chrm, ctOut$start, maxGap = 500)

## use probes in clusters with > 1 probe to determine background genes for pathway analysis
tab<-table(cl)
tab<-tab[which(tab > 1)]
backgroundDMRs<-GRanges(seqnames = ctOut$chrm[which(cl %in% names(tab))], strand = "*", ranges = IRanges(start = ctOut$start[which(cl %in% names(tab))], end = ctOut$end[which(cl %in% names(tab))]))
save(backgroundDMRs, file = "Analysis/CellType/BackgroundDMRProbes.rdata")
#segs <- getSegments(ctOut[,1], cl, cutoff=0.05, assumeSorted = FALSE)
#tabs<-regionFinder(ctOut[,1],newfData$chr, newfData$pos, cl, cutoff = 0.05)
nCor<-detectCores()
registerDoParallel(cores = nCor)

designMat<-model.matrix(formula( ~ pheno$Cell.type + pheno$Age + pheno$Sex))
## need to run twice to compare Neun+ to all others and then Sox10+ to all others
tab.neun<-bumphunter(celltypenormbeta, designMat, ctOut$chrm, as.numeric(ctOut$start), cl, coef = 3, cutoff = 0.1, nullMethod="bootstrap")
tab.irf8<-bumphunter(celltypenormbeta, designMat, ctOut$chrm, ctOut$start, cl, coef = 2,cutoff = 0.1, nullMethod="bootstrap")
tab.sox10<-bumphunter(celltypenormbeta, designMat, ctOut$chrm, ctOut$start, cl, coef = 4,cutoff = 0.1, nullMethod="bootstrap")
save(tab.neun, tab.sox10, tab.irf8,file="Analysis/CellType/BumpHunterBrainCellTypes.rdata")

## as bootstrapping takes too long/never seems to finish even with a handful of iterations look at overlap of significant DMPs in these regions

dmps<-GRanges(seqnames = ctOut$chrm, strand = "*", ranges = IRanges(start = ctOut$start, end = ctOut$end))
mcols(dmps)<-ctOut[,1:7]
neun.regions<-GRanges(seqnames = tab.neun$table$chr, strand = "*", ranges = IRanges(start = tab.neun$table$start, end = tab.neun$table$end))
mcols(neun.regions)<-tab.neun$table[,-c(1:3)]
sox10.regions<-GRanges(seqnames = tab.sox10$table$chr, strand = "*", ranges = IRanges(start = tab.sox10$table$start, end = tab.sox10$table$end))
mcols(sox10.regions)<-tab.sox10$table[,-c(1:3)]
irf8.regions<-GRanges(seqnames = tab.irf8$table$chr, strand = "*", ranges = IRanges(start = tab.irf8$table$start, end = tab.irf8$table$end))
mcols(irf8.regions)<-tab.irf8$table[,-c(1:3)]

## filter to regions with at least 2 sites
neun.regions<-neun.regions[neun.regions$L > 1]
sox10.regions<-sox10.regions[sox10.regions$L > 1]
irf8.regions<-irf8.regions[irf8.regions$L > 1]

## calculate proportion of probes in bump from cluster
summary(neun.regions$L/neun.regions$clusterL)
summary(sox10.regions$L/sox10.regions$clusterL)
summary(irf8.regions$L/irf8.regions$clusterL)

## for each region count significant DMPs
interDmpsNeun<-findOverlaps(dmps, neun.regions)
nDMPs<-table(subjectHits(interDmpsNeun)[which(mcols(dmps)[queryHits(interDmpsNeun), "NeuN:P"] < 9e-8)])
neun.regions$nDMPs<-as.numeric(nDMPs[match(1:length(neun.regions), names(nDMPs))])
neun.regions$nDMPs[is.na(neun.regions$nDMPs)]<-0

interDmpsSox10<-findOverlaps(dmps, sox10.regions)
nDMPs<-table(subjectHits(interDmpsSox10)[which(mcols(dmps)[queryHits(interDmpsSox10), "Sox10:P"] < 9e-8)])
sox10.regions$nDMPs<-as.numeric(nDMPs[match(1:length(sox10.regions), names(nDMPs))])
sox10.regions$nDMPs[is.na(sox10.regions$nDMPs)]<-0

interDmpsIrf8<-findOverlaps(dmps, irf8.regions)
nDMPs<-table(subjectHits(interDmpsIrf8)[which(mcols(dmps)[queryHits(interDmpsIrf8), "IRF8:P"] < 9e-8)])
irf8.regions$nDMPs<-as.numeric(nDMPs[match(1:length(irf8.regions), names(nDMPs))])
irf8.regions$nDMPs[is.na(irf8.regions$nDMPs)]<-0

summary(neun.regions$nDMPs)
summary(neun.regions$nDMPs/neun.regions$L)
length(which(neun.regions$nDMPs/neun.regions$L == 1))
summary(sox10.regions$nDMPs)
summary(sox10.regions$nDMPs/sox10.regions$L)
length(which(sox10.regions$nDMPs/sox10.regions$L == 1))
summary(irf8.regions$nDMPs)
summary(irf8.regions$nDMPs/irf8.regions$L)
length(which(irf8.regions$nDMPs/irf8.regions$L == 1))

## summarise size of bumps
summary(width(neun.regions))
summary(width(sox10.regions))
summary(width(irf8.regions))
summary(width(neun.regions[which(neun.regions$nDMPs > 0),]))
summary(width(sox10.regions[which(sox10.regions$nDMPs > 0),]))
summary(width(irf8.regions[which(irf8.regions$nDMPs > 0),]))
summary(width(neun.regions[which(neun.regions$nDMPs/neun.regions$L == 1),]))
summary(width(sox10.regions[which(sox10.regions$nDMPs/sox10.regions$L == 1),]))
summary(width(irf8.regions[which(irf8.regions$nDMPs/irf8.regions$L == 1),]))

## identify regions specific to each cell type
neun.specific<-neun.regions[unique(subjectHits(findOverlaps(setdiff(neun.regions, reduce(c(sox10.regions, irf8.regions))), neun.regions))),]
sox10.specific<-sox10.regions[unique(subjectHits(findOverlaps(setdiff(sox10.regions,reduce(c(neun.regions, irf8.regions))), sox10.regions))),]
irf8.specific<-irf8.regions[unique(subjectHits(findOverlaps(setdiff(irf8.regions,reduce(c(neun.regions, sox10.regions))), irf8.regions))),]

## noticed some specific regions are consecuative
neun.conseq<-unique(c(neun.specific[queryHits(findOverlaps(neun.specific, sox10.specific)),], neun.specific[queryHits(findOverlaps(neun.specific, irf8.specific)),]))
sox10.conseq<-unique(c(sox10.specific[subjectHits(findOverlaps(neun.specific, sox10.specific)),],sox10.specific[subjectHits(findOverlaps(irf8.specific, sox10.specific)),]))
irf8.conseq<-unique(c(irf8.specific[subjectHits(findOverlaps(neun.specific, irf8.specific)),],irf8.specific[subjectHits(findOverlaps(sox10.specific, irf8.specific)),]))

## how many DMPs are not located within

singleProbeCl<-names(table(cl))[which(table(cl) == 1)]
table(unique(cl[which(ctOut[,2] < 9e-8)]) %in% singleProbeCl)
table(unique(cl[which(ctOut[,4] < 9e-8)]) %in% singleProbeCl)
table(unique(cl[which(ctOut[,6] < 9e-8)]) %in% singleProbeCl)

## annotate with genes
txdb <- TxDb.Hsapiens.UCSC.hg38.knownGene
neunRegionAnno <- annotatePeak(neun.regions, tssRegion=c(-3000, 3000), TxDb=txdb)
sox10RegionAnno <- annotatePeak(sox10.regions, tssRegion=c(-3000, 3000), TxDb=txdb)
irf8RegionAnno <- annotatePeak(irf8.regions, tssRegion=c(-3000, 3000), TxDb=txdb)
save(neunRegionAnno, sox10RegionAnno, irf8RegionAnno, file = "Analysis/CellType/AnnotatedBumps.rdata")

## plot examples select those with the most DMPs
plotCols<-c("darkgreen", "darkblue", "darkmagenta", "deeppink", "darkgray") ## assumes celltypes are ordered alphabetically

axTrack <- GenomeAxisTrack()

pdf("Analysis/CellType/NeuNSpecificDMRs.pdf", width = 8, height = 8)
for(i in order(neun.specific$nDMPs, decreasing = TRUE)[1:10]){
	start <- as.numeric(start(neun.specific)[i])
	end <- as.numeric(end(neun.specific)[i])
	chr <- as.character(seqnames(neun.specific)[i])

	## add in window around region
	windowSize<-end-start
	start<-start-windowSize
	end<-end+windowSize
	probesInCluster<-rownames(newfData)[which(newfData$chr == chr & newfData$pos <= end+1000000 & newfData$pos >= start-1000000)] ### add in extra sites so that plot starts and finishes outside of plotting region

	idxTrack <- IdeogramTrack(genome="hg19", chromosome=chr)
	refGenes <- UcscTrack(genome="hg19", chromosome=chr, table="ncbiRefSeq", track = 'NCBI RefSeq',from=start, to=end, trackType="GeneRegionTrack", rstarts="exonStarts", rends="exonEnds", gene="name",
							symbol="name2", transcript="name", strand="strand", fill="#8282d2",
							  name="NCBI RefSeq", transcriptAnnotation="symbol", collapseTranscripts = "longest")
	cpgIslands <- UcscTrack(genome="hg19", chromosome=chr, track="cpgIslandExt", from=start, to=end,trackType="AnnotationTrack", start="chromStart", end="chromEnd", id="name",shape="box", fill="#006400", name="CpG Islands")

	cellMeans.granges<-GRanges(seqnames = newfData[probesInCluster, "chr"], strand = "*", ranges = IRanges(start = newfData[probesInCluster, "pos"], end = newfData[probesInCluster, "pos"]), cellMeans[probesInCluster, ])
	dTrack1<-DataTrack(range = cellMeans.granges, genome = "hg19", type = "p", chromosome=chr, name = "DNAm Mean", col = plotCols[1:4], type = "smooth", groups = c("Double -ve", "IRF8+ve", "NeuN+ve", "Sox10+ve"), lwd = 2, baseline = seq(0,1,0.2), col.baseline = "gray", lwd.baseline = 1)
	coeffs.granges<-GRanges(seqnames = newfData[probesInCluster, "chr"], strand = "*", ranges = IRanges(start = newfData[probesInCluster, "pos"], end = newfData[probesInCluster, "pos"]), ctOut[probesInCluster, c("IRF8:Coeff", "NeuN:Coeff", "Sox10:Coeff")])
	pvals.granges<-GRanges(seqnames = newfData[probesInCluster, "chr"], strand = "*", ranges = IRanges(start = newfData[probesInCluster, "pos"], end = newfData[probesInCluster, "pos"]), "IRF8:log10P" = -log10(ctOut[probesInCluster, c("IRF8:P")]),"NeuN:log10P" =  -log10(ctOut[probesInCluster, c("NeuN:P")]), "Sox10:log10P" = -log10(ctOut[probesInCluster, c("Sox10:P")]))
	dTrack2<-DataTrack(range = coeffs.granges, genome = "hg19", type = "p", chromosome=chr, name = "Reg. Coeffs", col = plotCols[c(2:4)], type = "smooth", groups = c("IRF8+ve", "NeuN+ve", "Sox10+ve"), lwd = 2, baseline = seq(-1,1,0.2), col.baseline = "gray", lwd.baseline = 1)
	dTrack3<-DataTrack(range = pvals.granges, genome = "hg19", type = "p", chromosome=chr, name = "-logP", ylim = c(0, max(mcols(pvals.granges)[,1])), groups = c("IRF8+ve", "NeuN+ve", "Sox10+ve"), baseline = 0, col.baseline = "gray", lwd.baseline = 1, col = plotCols[c(2:4)])
	ht <- HighlightTrack(trackList=list(axTrack, refGenes,cpgIslands, dTrack1,dTrack2, dTrack3), start=as.numeric(start(neun.specific)[i]), end = as.numeric(end(neun.specific)[i]),chromosome=chr)

	## set plot to just inside
	plotTracks(list(idxTrack, ht), from=start, to=end, showTitle=TRUE)
}
dev.off()

pdf("Analysis/CellType/Sox10DMRs.pdf", width = 8, height = 8)
for(i in order(sox10.regions$nDMPs, decreasing = TRUE)[1:10]){
	start <- tab.sox10$table$start[i]
	end <- tab.sox10$table$end[i]
	chr <- tab.sox10$table$chr[i]

	## add in window around region
	windowSize<-end-start
	start<-start-windowSize
	end<-end+windowSize

	idxTrack <- IdeogramTrack(genome="hg19", chromosome=chr)
	refGenes <- UcscTrack(genome="hg19", chromosome=chr, table="ncbiRefSeq", track = 'NCBI RefSeq',from=start, to=end, trackType="GeneRegionTrack", rstarts="exonStarts", rends="exonEnds", gene="name",
							symbol="name2", transcript="name", strand="strand", fill="#8282d2",
							  name="NCBI RefSeq", transcriptAnnotation="symbol", collapseTranscripts = "longest")
	cpgIslands <- UcscTrack(genome="hg19", chromosome=chr, track="cpgIslandExt", from=start, to=end,trackType="AnnotationTrack", start="chromStart", end="chromEnd", id="name",shape="box", fill="#006400", name="CpG Islands")

	probesInCluster<-rownames(newfData)[which(newfData$chr == chr & newfData$pos <= end & newfData$pos >= start)]

	cellMeans.granges<-GRanges(seqnames = newfData[probesInCluster, "chr"], strand = "*", ranges = IRanges(start = newfData[probesInCluster, "pos"], end = newfData[probesInCluster, "pos"]), cellMeans[probesInCluster, ])
	dTrack1<-DataTrack(range = cellMeans.granges, genome = "hg19", type = "p", chromosome=chr, name = "DNAm Mean", col = plotCols[1:4], type = "smooth", groups = c("Double -ve", "IRF8+ve", "NeuN+ve", "Sox10+ve"), lwd = 2, baseline = seq(0,1,0.2), col.baseline = "gray", lwd.baseline = 1)
	coeffs.granges<-GRanges(seqnames = newfData[probesInCluster, "chr"], strand = "*", ranges = IRanges(start = newfData[probesInCluster, "pos"], end = newfData[probesInCluster, "pos"]), ctOut[probesInCluster, c("IRF8:Coeff", "NeuN:Coeff", "Sox10:Coeff")])
	pvals.granges<-GRanges(seqnames = newfData[probesInCluster, "chr"], strand = "*", ranges = IRanges(start = newfData[probesInCluster, "pos"], end = newfData[probesInCluster, "pos"]), "IRF8:log10P" = -log10(ctOut[probesInCluster, c("IRF8:P")]),"NeuN:log10P" =  -log10(ctOut[probesInCluster, c("NeuN:P")]), "Sox10:log10P" = -log10(ctOut[probesInCluster, c("Sox10:P")]))
	dTrack2<-DataTrack(range = coeffs.granges, genome = "hg19", type = "p", chromosome=chr, name = "Reg. Coeffs", col = plotCols[c(2:4)], type = "smooth", groups = c("IRF8+ve", "NeuN+ve", "Sox10+ve"), lwd = 2, baseline = seq(-1,1,0.2), col.baseline = "gray", lwd.baseline = 1)
	dTrack3<-DataTrack(range = pvals.granges, genome = "hg19", type = "p", chromosome=chr, name = "-logP", ylim = c(0, max(mcols(pvals.granges)[,1])), groups = c("IRF8+ve", "NeuN+ve", "Sox10+ve"), baseline = 0, col.baseline = "gray", lwd.baseline = 1, col = plotCols[c(2:4)])
	ht <- HighlightTrack(trackList=list(axTrack, refGenes,cpgIslands, dTrack1,dTrack2, dTrack3), start=tab.sox10$table$start[i], end = tab.sox10$table$end[i],chromosome=chr)

	## set plot to just inside
	plotTracks(list(idxTrack, ht), from=start, to=end, showTitle=TRUE)
}
dev.off()

pdf("Analysis/CellType/IRF8DMRs.pdf", width = 8, height = 8)
for(i in order(irf8.regions$nDMPs, decreasing = TRUE)[1:10]){
	start <- tab.irf8$table$start[i]
	end <- tab.irf8$table$end[i]
	chr <- tab.irf8$table$chr[i]

	## add in window around region
	windowSize<-end-start
	start<-start-windowSize
	end<-end+windowSize

	idxTrack <- IdeogramTrack(genome="hg19", chromosome=chr)
	refGenes <- UcscTrack(genome="hg19", chromosome=chr, table="ncbiRefSeq", track = 'NCBI RefSeq',from=start, to=end, trackType="GeneRegionTrack", rstarts="exonStarts", rends="exonEnds", gene="name",
							symbol="name2", transcript="name", strand="strand", fill="#8282d2",
							  name="NCBI RefSeq", transcriptAnnotation="symbol", collapseTranscripts = "longest")
	cpgIslands <- UcscTrack(genome="hg19", chromosome=chr, track="cpgIslandExt", from=start, to=end,trackType="AnnotationTrack", start="chromStart", end="chromEnd", id="name",shape="box", fill="#006400", name="CpG Islands")

	probesInCluster<-rownames(newfData)[which(newfData$chr == chr & newfData$pos <= end & newfData$pos >= start)]

	cellMeans.granges<-GRanges(seqnames = newfData[probesInCluster, "chr"], strand = "*", ranges = IRanges(start = newfData[probesInCluster, "pos"], end = newfData[probesInCluster, "pos"]), cellMeans[probesInCluster, ])
	dTrack1<-DataTrack(range = cellMeans.granges, genome = "hg19", type = "p", chromosome=chr, name = "DNAm Mean", col = plotCols[1:4], type = "smooth", groups = c("Double -ve", "IRF8+ve", "NeuN+ve", "Sox10+ve"), lwd = 2, baseline = seq(0,1,0.2), col.baseline = "gray", lwd.baseline = 1)
	coeffs.granges<-GRanges(seqnames = newfData[probesInCluster, "chr"], strand = "*", ranges = IRanges(start = newfData[probesInCluster, "pos"], end = newfData[probesInCluster, "pos"]), ctOut[probesInCluster, c("IRF8:Coeff", "NeuN:Coeff", "Sox10:Coeff")])
	pvals.granges<-GRanges(seqnames = newfData[probesInCluster, "chr"], strand = "*", ranges = IRanges(start = newfData[probesInCluster, "pos"], end = newfData[probesInCluster, "pos"]), "IRF8:log10P" = -log10(ctOut[probesInCluster, c("IRF8:P")]),"NeuN:log10P" =  -log10(ctOut[probesInCluster, c("NeuN:P")]), "Sox10:log10P" = -log10(ctOut[probesInCluster, c("Sox10:P")]))
	dTrack2<-DataTrack(range = coeffs.granges, genome = "hg19", type = "p", chromosome=chr, name = "Reg. Coeffs", col = plotCols[c(2:4)], type = "smooth", groups = c("IRF8+ve", "NeuN+ve", "Sox10+ve"), lwd = 2, baseline = seq(-1,1,0.2), col.baseline = "gray", lwd.baseline = 1)
	dTrack3<-DataTrack(range = pvals.granges, genome = "hg19", type = "p", chromosome=chr, name = "-logP", ylim = c(0, max(mcols(pvals.granges)[,1])), groups = c("IRF8+ve", "NeuN+ve", "Sox10+ve"), baseline = 0, col.baseline = "gray", lwd.baseline = 1, col = plotCols[c(2:4)])
	ht <- HighlightTrack(trackList=list(axTrack, refGenes,cpgIslands, dTrack1,dTrack2, dTrack3), start=tab.irf8$table$start[i], end = tab.irf8$table$end[i],chromosome=chr)

	## set plot to just inside
	plotTracks(list(idxTrack, ht), from=start, to=end, showTitle=TRUE)
}
dev.off()


######### internal functions #########

##' @title tss
##'
##' @param x count matrix (each OTU in one row)
##' @description Calculate the relative abundances by dividing the total abundance (Total Sum Sacling)
##' @author Chenhao Li, Gerald Tan, Niranjan Nagarajan
tss <- function(x){ apply(x, 2, function(x) x/sum(x)) }

##' @title css
##'
##' @param m matrix of data (variables in columns, measurements in rows)
##' @param p quantile used for normalization (default: 0.5)
##' @description Function to perform cumulative sum scaling (CSS)
##' @author Chenhao Li, Gerald Tan, Niranjan Nagarajan
css <- function(m, p=0.5){
  m[m==0] <- NA
  ## find the quantile in each sample
  quant <- apply(m,1,function(x) quantile(x, p=p, na.rm = TRUE))
  ## calculate normalization factor
  f <- rowSums(m*sweep(m, 1, quant, '<='),na.rm = TRUE)
  nf <- f/exp(mean(log(f)))
  dat.css <- sweep(m,1,nf, '/')
  return(list("normCounts" = dat.css, "normFactors"=nf))
}

##' @title infer
##'
##' @param Y response
##' @param X predictors
##' @param method blasso or lm
##' @param intercept whether to include the intercept
##' @param seed seed
##' @param alpha the alpha parameter for elastic net (1:lasso [default], 0:ridge)
##' @param lambda.init user provides initial lambda values
##' @param lambda.choice 1: use lambda.1se for LASSO, 2: use lambda.min for LASSO, a number between (0, 1): this will select a lambda according to (1-lambda.choice)*lambda.min + lambda.choice*lambda.1se
##' @param nfolds number of folds for glmnet cross-validation
##' @import glmnet
##' @description Infer parameters with blasso
##' @author Chenhao Li, Gerald Tan, Niranjan Nagarajan
infer <- function(Y, X, method='glmnet', intercept=FALSE, seed=0, alpha=1, nfolds=5,
                  lambda.init=NULL, lambda.choice=1){
  lambda.lower <- 1e-9
  lambda.upper <- 1
  set.seed(seed)
  if(method=='lm'){
    return(as.numeric(coef(lm(Y~X+0))))
  }
  if(method=='glmnet'){
    penalty <- c(0,rep(1, ncol(X)-1))
    maxmin <- median(Y) + 5 * IQR(Y) %*% c(1,-1)
    idx <- Y <= maxmin[1] & Y >= maxmin[2]
    if(is.null(lambda.init)){
      ## select a lambda range first
      lambda.init <- rev(exp(seq(log(lambda.lower), log(lambda.upper), length.out = 50)))
      fit <- cv.glmnet(X[idx,], Y[idx], intercept=intercept, lambda=lambda.init, nfolds=nfolds,
                       penalty.factor=penalty, alpha=alpha)
      lambda <- rev(exp(seq(
        max(log(fit$lambda.min/10), log(lambda.lower)),
        min(log(fit$lambda.min*10), log(lambda.upper)),
        length.out = 50)))
    }else{
      ## adjust lambda by up to 50%
      lambda <- rev(exp(seq(
        max(log(lambda.init/2), log(lambda.lower)), ## bounded by the min lambda
        min(log(lambda.init*1.5), log(lambda.upper)), ## bounded by the max lambda
        length.out = 20)))
    }

    fit <- cv.glmnet(X[idx,], Y[idx], intercept=intercept, lambda=lambda,  nfolds=nfolds,
                     penalty.factor=penalty, alpha=alpha)
    if(lambda.choice == 1){
      s = fit$lambda.1se
    }else if(lambda.choice == 2){
      s = fit$lambda.min
    }else{
      s = (1-lambda.choice)*fit$lambda.min + lambda.choice*fit$lambda.1se
    }
    coefs <- coef(fit, s=s)[-1]
    e <- as.numeric((Y-(X %*% coefs)[,1]))
    rm(.Random.seed, envir=.GlobalEnv)
    return(c(coefs, e, s))
  }
}

# ##' @title boot_stat
# ##'
# ##' @param data data in the format of cbind(Y, X)
# ##' @param indices indices for bootstrapping
# ##' @description bootstrapping the inference process
# ##' @author Chenhao Li, Gerald Tan, Niranjan Nagarajan
# bs <- function(data, indices) {
#     X_s <- data[indices, -1] # allows boot to select sample
#     Y_s <- data[indices, 1]
#     fit <- infer(Y_s, X_s)
#     return(fit[1:ncol(X_s)])
# }

# ##' @title func.E.boot
# ##'
# ##' @param dat.tss relative abundances matrix (each OTU in one row)
# ##' @param sample.filter filter out samples contain outliers in Y
# ##' @param m estimated biomass values
# ##' @param ncpu number of CPUs (default: 4)
# ##' @importFrom boot boot
# ##' @importFrom doParallel registerDoParallel
# ##' @import foreach
# ##' @description bootstrapped E-step
# ##' @author Chenhao Li, Gerald Tan, Niranjan Nagarajan
# func.E.boot <- function(dat.tss, m, sample.filter, ncpu=4, ...){
#     registerDoParallel(ncpu)
#     p <- nrow(dat.tss)
#     res <- foreach(i=1:p, .combine=rbind) %do% {
#         message(paste0("Bootstrapping for species ", i))
#         fil <- dat.tss[i,]!=0 & !sample.filter[i,]
#         X <- t(rbind(1/m, dat.tss[-i,])[,fil])
#         Y <- dat.tss[i, fil]
#         theta <- rep(0, p+1)
#         stab <- rep(1, p)
#         theta[i+1] <- -1 ## -beta_{ii}/beta_{ii}
#         tmp <- boot(data=cbind(Y,X), statistic=bs, R=100)
#         theta[-(i+1)] <- apply(tmp$t, 2, median)
#         stab[-(i)] <- (colSums(tmp$t!=0)/tmp$R)[-1]
#         c(theta, stab)
#     }
#     list(a=res[,1], b=postProcess(res[,1:p+1]), stab=res[,(p+2):ncol(res)])
# }

# ##' @title func.E.stab
# ##'
# ##' @param dat.tss relative abundances matrix (each OTU in one row)
# ##' @param sample.filter filter out samples contain outliers in Y
# ##' @param m estimated biomass values
# ##' @param ncpu number of CPUs (default: 4)
# ##' @param perc percentage of samples to take for each iteration
# ##' @param niter number of iterations to run
# ##' @param ... additional parameters for beemStatic:::infer
# ##' @importFrom doParallel registerDoParallel
# ##' @import foreach
# ##' @description E-step with subsampling to estimate stability
# ##' @author Chenhao Li, Gerald Tan, Niranjan Nagarajan
# func.E.stab <- function(dat.tss, m, sample.filter, ncpu=4, perc=0.6, niter=100, ...){
#   registerDoParallel(ncpu)
#   p <- nrow(dat.tss)
#   res <- foreach(i=1:p, .combine=rbind) %do% {
#     message(paste0("Resampling for species ", i))
#     fil <- dat.tss[i,]!=0 & !sample.filter[i,]
#     X <- t(rbind(1/m, dat.tss[-i,])[,fil])
#     Y <- dat.tss[i, fil]
#     theta <- rep(0, p+1)
#     stab <- rep(1, p)
#     theta[i+1] <- -1 ## -beta_{ii}/beta_{ii}
#     tmp <- (sapply(1:niter, function(x) sub_stat(data=cbind(Y,X), perc) ))
#     theta[-(i+1)] <- apply(tmp, 1, median)
#     stab[-(i)] <- (rowSums(tmp!=0)/niter)[-1]
#     c(theta, stab)
#   }
#   list(a=res[,1], b=postProcess(res[,1:p+1]), stab=res[,(p+2):ncol(res)])
# }

# ##' @title sub_stat
# ##'
# ##' @param data data in the format of cbind(Y, X)
# ##' @param perc percentage of samples to take for each iteration
# ##' @param ... additional parameters for beemStatic:::infer
# ##' @description Resampling for the inference process
# ##' @author Chenhao Li, Gerald Tan, Niranjan Nagarajan
# sub_stat <- function(data, perc, ...) {
#   n <- nrow(data)
#   indices <- sample(1:n, n*perc)
#   X_s <- data[indices, -1]
#   Y_s <- data[indices, 1]
#   fit <- infer(Y_s, X_s)
#   return(fit[1:ncol(X_s)])
# }

##' @title norm
##'
##' @param a estimated growth rate values (scaled by self-interaction)
##' @param b estimated interaction matrix (scaled by self-interaction)
##' @param c estimated external perturbation effect matrix (scaled by self-interaction) (default: NULL)
##' @param s external perturbation presence vector in one sample
##' @param x relative abundances in one sample
##' @description estimate biomass with linear regression
##' @author Chenhao Li, Gerald Tan, Niranjan Nagarajan
norm <- function(a, b, c = NULL, s = NULL, x){
  if (!is.null(c) || !is.null(s)) {
    perturbation.eff <- c %*% s
    res <- -(a + perturbation.eff) / (b %*% x)
  } else {
    res <- -a / (b %*% x)
  }
  res <- res[x!=0]
  if(all(res < 0)){
    m <- abs(max(res))
  } else{
    m <- median(res[res>0])
  }
  if (!is.null(c) || !is.null(s)) {
    err <- (m * (b %*% x) + a + perturbation.eff)[,1]/a
  } else {
    err <- (m * (b %*% x) + a)[,1]/a
  }
  err[x==0] <- 0
  c(m, err)

}

##' @title entropy
##'
##' @param v a vector input (treated as binary, i.e. zero/non-zero)
##' @description Compute the entropy of a binary vector
##' @author Chenhao Li, Gerald Tan, Niranjan Nagarajan
entropy <- function(v){
  p <- sum(v == 0)/length(v)
  ifelse(p==0 || p==1, 0, -(p*log2(p) + (1-p)*log2(1-p)) )
}

##' @title func.E
##'
##'
##' @param dat.tss relative abundances matrix (each OTU in one row)
##' @param external.perturbation external perturbation presence matrix *1/m (each perturbation in one row, each sample in one column) (Default: NULL)
##' @param sample.filter filter out samples contain outliers in Y
##' @param lambda.inits initial lambda values
##' @param m estimated biomass values (1 X no. of samples) matrix
##' @param ncpu number of CPUs (default: 4)
##' @param center center data or not
##' @param ... additional parameters for `beemStatic::infer`
##' @importFrom doParallel registerDoParallel
##' @import foreach
##' @description E-part of BEEM, estimate model parameters with inferred m
##' @author Chenhao Li, Gerald Tan, Niranjan Nagarajan
func.E <- function(dat.tss, external.perturbation = NULL, m, sample.filter, lambda.inits=NULL, ncpu=4, center=FALSE, ...){
  ## infer parameter for each OTU
  registerDoParallel(ncpu)
  p <- nrow(dat.tss)
  if (!is.null(external.perturbation)) {
    k <- nrow(external.perturbation) #Number of external perturbations
  }
  res <- foreach(i=1:p, .combine=rbind) %dopar% {
    fil <- dat.tss[i,]!=0 & !sample.filter[i,]
    X <- t(rbind(1/m, dat.tss[-i,])[,fil])
    Y <- dat.tss[i, fil]
    if(center){
      Y <- Y-mean(Y)
      X <- X-rowMeans(X)
    }
    theta <- rep(0, p+1)
    theta[i+1] <- -1 ## -beta_{ii}/beta_{ii}
    if (!is.null(external.perturbation)) {
      perturbation.coefficients <- rep(0, k) #Creating a vector to store perturbation coefficients
      external.perturbation.filtered <- external.perturbation[,fil] #Filtering out the external perturbations corresponding to samples which were filtered out
      X <- cbind(X, t(external.perturbation.filtered))
    }

    tmp <- infer(Y,X, lambda.init=lambda.inits[i], ...)

    if (!is.null(external.perturbation)) {
      theta[-(i+1)] <- tmp[1:p]
      perturbation.coefficient <- tmp[seq(p+1,p+k)]
      e <- rep(NA, ncol(dat.tss))
      e[fil] <- tmp[(p+k+1):(length(tmp)-1)]
      lambda <- tmp[length(tmp)]
      c(theta, perturbation.coefficient, e, lambda)
    } else {
      theta[-(i+1)] <- tmp[1:p]
      e <- rep(NA, ncol(dat.tss))
      e[fil] <- tmp[(p+1):(length(tmp)-1)]
      lambda <- tmp[length(tmp)]
      c(theta, e, lambda)
    }
  }

  ## check if there is not enough information
  uncertain <- foreach(i=1:p, .combine=rbind) %dopar% {
    fil <- dat.tss[i,]!=0
    X <- dat.tss[, fil]
    apply(X, 1, entropy)
    ## abs(rowSums(X!=0)/ncol(X) - 0.5) ## non-zero entries
  }
  if (!is.null(external.perturbation)) {
    list(a=res[,1], b=postProcess_species(res[,1:p+1]),
         perturbation.coefficients = postProcess_perturbation(matrix(res[,seq(p+2,p+k+1)], ncol = k), res[,1]),
         e=res[,(p+k+2):(ncol(res)-1)],
         uncertain=uncertain, lambdas=res[,ncol(res)])
  } else {
    list(a=res[,1], b=postProcess_species(res[,1:p+1]), e=res[,(p+2):(ncol(res)-1)],
         uncertain=uncertain, lambdas=res[,ncol(res)])
  }
}

##' @title func.M
##'
##' @param dat.tss relative abundance matrix (each OTU in one row)
##' @param a estimated growth rate values (scaled by self-interaction)
##' @param b estimated interaction matrix (scaled by self-interaction)
##' @param c estimated external perturbation effect matrix (scaled by self-interaction)
##' @param perturbation.presence estimated external perturbation presence (not to be multiplied by 1/m) (each perturbation in one row, each sample in one column)
##' @param ncpu number of CPUs (default: 4)
##' @importFrom doParallel registerDoParallel
##' @import foreach
##' @description M-part of BEEM, estimate biomass with inferred model parameters
##' @author Chenhao Li, Gerald Tan, Niranjan Nagarajan
func.M <- function(dat.tss, a, b, c = NULL, perturbation.presence=NULL, ncpu=4){
  registerDoParallel(ncpu)
  foreach(i=1:ncol(dat.tss), .combine=rbind) %dopar%{
    x <- dat.tss[,i]
    if (!is.null(c) || !is.null(perturbation.presence)) {
      s <- perturbation.presence[,i]
      norm(a, b, c = c, s = s, x)
    } else {
      norm(a, b, c = NULL, s = NULL, x)
    }
  }
}

##' @title preProcess
##'
##' @param dat OTU count/relative abundance matrix (each OTU in one row)
##' @param dev dev * IQR from median will be filtered out (default: 0, nothing to remove)
##' @description pre-process data
##' @author Chenhao Li, Gerald Tan, Niranjan Nagarajan
preProcess <- function(dat, dev=0){
  ## filter out species abundances that are too low
  detection_limit <- 1e-4
  dat <- tss(dat)
  dat[dat<detection_limit] <- 0
  ##rowMed <- apply(dat, 1, function(x) median(x[x!=0]))
  ##rowIQR <- apply(dat, 1, function(x) IQR(x[x!=0]))
  rowMAD <- apply(dat, 1, function(x) mad(x[x!=0]))
  ## too close to zero
  sample.filter <- dat/rowMAD < dev
  return(list(tss=dat, sample.filter=sample.filter))
}

##' @title postProcess_species
##'
##' @param b scaled interaction matrix (diagonal/self-interaction is -1)
##' @param dev scale of the interaction < dev * 1, the value will be set to 0
##' @description post-process parameters to remove small entries
##' @author Chenhao Li, Gerald Tan, Niranjan Nagarajan
postProcess_species <- function(b, dev=1e-5){
  b[abs(b)<dev] <- 0
  b
}

##' @title postProcess_perturbation
##'
##' @param b scaled perturbation matrix (comparing to growthrate)
##' @param dev scale of the interaction < dev * 1, the value will be set to 0
##' @description post-process parameters to remove small entries
##' @author Chenhao Li, Gerald Tan, Niranjan Nagarajan
postProcess_perturbation <- function(c, growth.rate, dev=1e-5){
  c[abs(c/growth.rate)<dev] <- 0
  c
}

##' @title formatOutput
##' @importFrom reshape2 melt
##' @param a scaled growth rates
##' @param b scaled interaction matrix
##' @param c scaled external perturbation matrix (default: NULL)
##' @param vnames species names
##' @param pnames external perturbation names (default: NULL)
##' @description Function to convert parameter vector a and matrix b to MDSINE's output format
##' @author Chenhao Li, Gerald Tan, Niranjan Nagarajan
formatOutput <- function(a, b, c = NULL, vnames, pnames = NULL){
  p <- length(a)
  param <- data.frame(parameter_type=c(rep("growth_rate",p), rep("interaction",p*p)))
  param$source_taxon <- c(rep(NA,p),rep(vnames, each = p))
  param$target_taxon <- vnames
  param$value <- 0
  param$significance <- NA
  tmp.b <- melt(t(b))
  tmp.b <- tmp.b[order(tmp.b$Var1),]
  param$value <- c(a, tmp.b$value)
  if (!is.null(c) || !is.null(pnames)) {
    q <- ncol(c) #Finding the number of external perturbationss
    ext.perturb <- data.frame(parameter_type=c(rep("external perturbations", p*q)))
    ext.perturb$source_taxon <- c(rep(NA, p*q))
    ext.perturb$target_taxon <- vnames
    ext.perturb$value <- 0
    ext.perturb$significance <- NA
    tmp.c <- melt(c)
    ext.perturb$value <- c(tmp.c$value)
    param <- rbind(param, ext.perturb)
  }
  param$external_perturbation <- c(rep(NA,p), rep(NA,p*p), rep(pnames, each = p)) #Creating the vector for external perturbations
  param

}

##' @title detectBadSamples
##'
##' @param err the errors estimated from regression
##' @param threshold threshold to filter out samples
##' @author Chenhao Li, Gerald Tan, Niranjan Nagarajan
detectBadSamples <- function(err, threshold){
  score <- (err - median(err, na.rm = TRUE))/IQR(err, na.rm = TRUE)
  score[is.na(score)] <- Inf
  return(score > threshold)
}

######### Exported functions #########

##' @title beem2param
##'
##' @param beem a BEEM object
##' @description extract parameter estimates from a BEEM object
##' @export
##' @author Chenhao Li, Gerald Tan, Niranjan Nagarajan
beem2param <- function(beem){
  p <- ncol(beem$err.m) #Number of species
  num.perturb <- (nrow(beem$trace.p) - p - p*p)/p
  tmp <- beem$trace.p[, ncol(beem$trace.p)]
  a.est <- tmp[1:p]
  b.est <- matrix(tmp[-c(1:p)], p, p)
  if (num.perturb == 0) {
    return (list(a.est=a.est, b.est=b.est))
  } else {
    c.est <- matrix(tmp[(nrow(beem$trace.p) - (num.perturb*p - 1)):nrow(beem$trace.p)],
                    ncol = num.perturb)
    return (list(a.est=a.est, b.est=b.est, c.est=c.est))
  }
}

##' @title beem2biomass
##'
##' @param beem a BEEM object
##' @description extract biomass estimates from a BEEM object
##' @export
##' @author Chenhao Li, Gerald Tan, Niranjan Nagarajan
beem2biomass <- function(beem){
  niter <- ncol(beem$trace.m)
  return (beem$trace.m[,niter])
}

##' @title func.EM
##'
##' @param dat OTU count/relative abundance matrix (each OTU in one row)
##' @param external.perturbation external perturbation presence matrix (each perturbation in one row, each sample in one column) (Default: NULL)
##' @param ncpu number of CPUs (default: 4)
##' @param scaling a scaling factor to keep the median of all biomass constant (default: 1000)
##' @param dev deviation of the error (for one sample) from the model to be excluded (default: Inf - all the samples will be considered)
##' @param m.init initial biomass values (default: use CSS normalization)
##' @param max.iter maximal number of iterations (default 30)
##' @param warm.iter number of iterations to run before removing any samples and stop adjusting lambda (default: run until convergence and start to remove samples)
##' @param lambda.choice 1: use lambda.1se for LASSO, 2: use lambda.min for LASSO, a number between (0, 1): this will select a lambda according to (1-lambda.choice)*lambda.min + lambda.choice*lambda.1se
##' @param resample number of iterations to resample the data to compute stability of the interaction parameters (default: 0 - no resampling)
##' @param alpha the alpha parameter for the Elastic Net model (1-LASSO [default], 0-RIDGE)
##' @param refresh.iter refresh the removed samples every X iterations (default: 1)
##' @param epsilon convergence threshold (in relative difference): uqn of the relative error in biomass changes (default 1e-3)
##' @param verbose print out messages
##' @param debug output debugging information (default FALSE)
##' @description Iteratively estimating scaled parameters and biomass
##' @export
##' @author Chenhao Li, Gerald Tan, Niranjan Nagarajan
func.EM <- function(dat, external.perturbation = NULL, ncpu=4, scaling=1000, dev=Inf, m.init=NULL,
                    max.iter=30, warm.iter=NULL, lambda.choice=1, resample=0,
                    alpha=1, refresh.iter=1, epsilon=1e-3,
                    debug=FALSE, verbose=TRUE){

  ## pre-processing
  dat.init <- preProcess(dat, dev=0)
  dat.tss <- dat.init$tss
  spNames <- rownames(dat)

  ## ensure valid samples
  temp <- colSums(dat.tss, na.rm = TRUE) == 0
  if(any(temp)){
    stop(paste0('Sample ', which(temp), ' has zero total abudance...'))
  }

  ## initialization
  sample.filter.iter <- dat.init$sample.filter
  tmp <- css(t(dat.tss))$normFactors
  if(is.null(m.init)) {
    m.iter <- scaling * tmp/median(tmp)
  }else{
    m.iter <- scaling * m.init/median(m.init)
  }
  trace.m <- matrix(m.iter)
  trace.p <- apply(expand.grid(spNames, spNames), 1, function(x) paste0(x[2], '->', x[1]))
  trace.p <- c(paste0(NA, '->',spNames), trace.p)
  if (!is.null(external.perturbation)) {
    external.perturbation.Names <- rownames(external.perturbation)
    trace.p.external.perturbation <- apply(expand.grid(spNames, external.perturbation.Names), 1, function(x) paste0(x[2], '->', x[1]))
    trace.p <- append(trace.p, trace.p.external.perturbation)
  }
  trace.p <- data.frame(name=trace.p)
  trace.lambda <- data.frame(spNames)
  lambda.inits <- NULL
  ## flags
  remove_non_eq <- FALSE
  removeIter <- 0

  ## constants
  m1 <- matrix(rep(1,nrow(sample.filter.iter)))
  ## EM
  for(iter in 1:max.iter){
    if(verbose) message(paste0("############# Run for iteration ", iter,": #############"))
    if(verbose) message("E-step: estimating scaled parameters...")
    if(iter>=2) lambda.inits <- lambdas
    if(is.null(external.perturbation)){
      ext.pert <- NULL
    }else{
      ext.pert <- 1/m.iter * external.perturbation
    }
    tmp.p <- func.E(dat.tss, external.perturbation = ext.pert,
                    m.iter, sample.filter.iter, ncpu=ncpu, method='glmnet',
                    lambda.inits=lambda.inits, alpha=alpha)
    err.p <- tmp.p$e
    lambdas <- tmp.p$lambdas
    uncertain <- tmp.p$uncertain
    if(debug){
      print(rowSums(tmp.p$b!=0))
      plot(1-rowMeans(err.p^2, na.rm = TRUE)/apply(dat.tss[,], 1, function(x) var(x[x!=0], na.rm=TRUE) ))
      ##abline(h=0.5)
    }

    if(verbose) message("M-step: estimating biomass...")
    if (!is.null(external.perturbation)) {
      tmp.m <- func.M(dat.tss, tmp.p$a, tmp.p$b, c = tmp.p$perturbation.coefficients, perturbation.presence = external.perturbation, ncpu)
    } else {
      tmp.m <- func.M(dat.tss, tmp.p$a, tmp.p$b, c = NULL, perturbation.presence = NULL, ncpu)
    }
    m.iter <- tmp.m[,1]
    err.m <- tmp.m[,-1]

    if(remove_non_eq){
      ## clear up removed samples every X iterations
      if (iter %% refresh.iter == 0 ) {
        sample.filter.iter <- dat.init$sample.filter
      }
      bad.samples <- detectBadSamples(apply(err.p^2, 2, median, na.rm = TRUE), dev)
      sample.filter.iter <- (m1 %*% bad.samples)>0  | sample.filter.iter
      if(verbose) message(paste0("Number of samples removed (detected to be non-static): ",
                                 sum(colSums(sample.filter.iter)>0)))
      removeIter <- removeIter + 1
    }
    m.fil <- colSums(sample.filter.iter)==0
    m.iter <- m.iter*scaling/median(m.iter[m.fil], na.rm=TRUE)
    trace.m <- cbind(trace.m, m.iter)
    if (!is.null(external.perturbation)) {
      trace.p <- cbind(trace.p, formatOutput(tmp.p$a, tmp.p$b, c = tmp.p$perturbation.coefficients,
                                             spNames, pnames = external.perturbation.Names)$value)
    } else {
      trace.p <- cbind(trace.p, formatOutput(tmp.p$a, tmp.p$b, c = NULL, spNames, pnames = NULL)$value)
    }

    trace.lambda <- cbind(trace.lambda, lambdas)
    summary.m <- apply(trace.m[m.fil, ], 1, function(x) median(rev(x)[1:min(5,length(x))]))
    #criterion <- quantile(abs((trace.m[m.fil, iter+1] - summary.m)/summary.m), 1)<epsilon
    criterion <- median(abs((trace.m[m.fil, iter+1] - trace.m[m.fil, iter])/trace.m[m.fil, iter]))<epsilon #0.0025

    if (!is.null(warm.iter) && iter > warm.iter && !remove_non_eq){
      if(verbose) message("Start to detect and remove bad samples...")
      remove_non_eq <- TRUE
    }
    if (iter > 5 && !remove_non_eq && criterion && is.finite(dev)) {
      if(verbose) message("Converged and start to detect and remove bad samples...")
      remove_non_eq <- TRUE
    }
    if (((removeIter > 5 && remove_non_eq) || is.infinite(dev)) && criterion) {
      if(verbose) message("Converged!")
      break
    }
  }
  res.resample <- NULL

  if(resample!=0){
    if(verbose) message("Estimating stability...")
    ##res.resample <- func.E.stab(dat.tss, m.iter, sample.filter.iter, ncpu=ncpu, alpha=alpha)
    if (!is.null(external.perturbation)) {
      res.resample <- resample.EM(dat[, m.fil], external.perturbation = external.perturbation[, m.fil], m=m.iter[m.fil],
                                  perc=0.6, res.iter=resample,
                                  ncpu=ncpu, scaling=scaling, dev=dev, refresh.iter=refresh.iter, alpha=alpha,
                                  max.iter=20, warm.iter=0, resample=0, debug=FALSE, verbose=FALSE)
    } else {
      res.resample <- resample.EM(dat[, m.fil], external.perturbation = NULL, m=m.iter[m.fil],
                                  perc=0.6, res.iter=resample,
                                  ncpu=ncpu, scaling=scaling, dev=dev, refresh.iter=refresh.iter, alpha=alpha,
                                  max.iter=20, warm.iter=0, resample=0, debug=FALSE, verbose=FALSE)
    }
    res.resample$a.summary <- apply(res.resample$res.a, 1, median)
    res.resample$b.summary <- matrix(apply(res.resample$res.b, 1, function(x) median(x)), nrow(dat))
    res.resample$b.stab <- matrix(rowSums(res.resample$res.b != 0)/ncol(res.resample$res.a), nrow(dat))
    if (!is.null(external.perturbation)) {
      res.resample$c.summary <- apply(res.resample$res.c, 1, function(x) median(x))
      res.resample$c.stab <- matrix(rowSums(res.resample$res.c != 0)/ncol(res.resample$res.a), nrow(dat))
    }
  }
  list(trace.m=trace.m, trace.p=trace.p, err.m=err.m, err.p=err.p,
       b.uncertain = uncertain, trace.lambda=trace.lambda,
       resample=res.resample,
       dev=dev,
       sample2rm = which(!m.fil))
}

##' @title resample.EM
##'
##' @param data data in the format of cbind(Y, X)
##' @param external.perturbation external perturbation presence matrix (each perturbation in one row, each sample in one column) (Default: NULL)
##' @param m biomass initialization
##' @param perc percentage of samples to take for each iteration
##' @param res.iter number of resample iteration
##' @param ... additional parameters for beemStatic::func.EM
##' @description Resampling for the inference process
##' @export
##' @author Chenhao Li, Gerald Tan, Niranjan Nagarajan
resample.EM <- function(data, external.perturbation = NULL, m, perc, res.iter, ...) {
  n <- ncol(data)
  p <- nrow(data)
  res_p <- data.frame(matrix(NA, p*(p+1), res.iter))
  res_m <- data.frame(matrix(NA, n, res.iter))

  for(i in 1:res.iter){
    message(paste0("#### Resample iteration: ", i, " #####"))
    indices <- sort(sample(1:n, n*perc))
    if (!is.null(external.perturbation)) {
      num.perturb <- nrow(external.perturbation)
      tmp <- func.EM(data[, indices], external.perturbation = external.perturbation[, indices], m.init=m[indices], ...)
    } else {
      tmp <- func.EM(data[, indices], external.perturbation = NULL, m.init=m[indices], ...)
    }
    res_p[,i] <- tmp$trace.p[, ncol(tmp$trace.p)]
    res_m[indices,i] <- tmp$trace.m[, ncol(tmp$trace.m)]
  }
  if (!is.null(external.perturbation)) {
    list(res.a = res_p[1:p,], res.b = res_p[(p+1):(nrow(res_p)-num.perturb*p),], res.c = res_p[(nrow(res_p)-num.perturb*p + 1):nrow(res_p),], res.m = res_m)
  } else {
    list(res.a = res_p[1:p,], res.b = res_p[-c(1:p),], res.m = res_m)
  }

  list(res.a = res_p[1:p,], res.b = res_p[-c(1:p),])
}

##' @title predict_beem
##'
##' @param dat.new OTU count/relative abundance matrix (each OTU in one row)
##' @param beem output of the EM algorithm
##' @param dev deviation of the error (for one sample) from the model to be excluded
##' @param ncpu number of CPUs (default: 4)
##' @param pert.new external perturbation presence matrix (each perturbation in one row, each sample in one column) (Default: NULL)
##' @importFrom doParallel registerDoParallel
##' @description Use a trained BEEM-static model to predict biomass, deviation from steady states and violation of model assumption
##' @export
##' @author Chenhao Li, Gerald Tan, Niranjan Nagarajan
predict_beem <- function(dat.new, beem, dev, ncpu=4, pert.new = NULL){
  ### currently not ready for an S3method yet
  param <- beem2param(beem)
  dat.new.tss <- tss(dat.new)
  if (!is.null(pert.new)){
    tmp.m <- func.M(dat.new.tss, param$a.est, param$b.est, c = param$c.est, perturbation.presence = pert.new, ncpu = 1)
  } else {
    tmp.m <- func.M(dat.new.tss, param$a.est, param$b.est, ncpu=ncpu)
  }
  m.pred <- tmp.m[,1]
  err.m.pred <- tmp.m[,-1]

  registerDoParallel(ncpu)
  p <- nrow(dat.new.tss)
  e <- foreach(i=1:p, .combine=rbind) %dopar% {
    X <- t(rbind(1/m.pred, dat.new.tss[-i,]))
    Y <- dat.new.tss[i, ]
    coefs <- c(param$a.est[i], param$b.est[i,-i])
    if(!is.null(param$c.est)){
        pert <- t(pert.new)
        coefs_pert <- param$c.est[i,]
        as.numeric(Y - (X %*% coefs)[,1] - (1/m.pred) * ((pert %*% coefs_pert)[,1]) )
    }else{
        as.numeric((Y - (X %*% coefs)[,1] ) )
    }
  }
  isBad <- detectBadSamples(apply(e^2, 2, median, na.rm = TRUE), dev)
  return(list(biomass.pred=m.pred, dev.from.eq=t(err.m.pred), isBad=isBad))
}


softmax <- function (x) {
    m <- max(x)
    exps <- exp(x - m)
    s <- sum(exps)
    return(exps / s)
}
score <- function(input) {
    if ((input[3]) > (3.1500000000000004)) {
        var0 <- -1.1986122886681099
    } else {
        if ((input[2]) > (3.35)) {
            var0 <- -0.8986122886681098
        } else {
            var0 <- -0.9136122886681098
        }
    }
    if ((input[3]) > (3.1500000000000004)) {
        if ((input[3]) > (4.450000000000001)) {
            var1 <- -0.09503010837903424
        } else {
            var1 <- -0.09563272415214283
        }
    } else {
        if ((input[2]) > (3.35)) {
            var1 <- 0.16640323607832397
        } else {
            var1 <- 0.15374604217339707
        }
    }
    if ((input[3]) > (1.8)) {
        if ((input[4]) > (1.6500000000000001)) {
            var2 <- -1.2055899476674514
        } else {
            var2 <- -0.9500445227622534
        }
    } else {
        var2 <- -1.2182214705715104
    }
    if ((input[4]) > (0.45000000000000007)) {
        if ((input[4]) > (1.6500000000000001)) {
            var3 <- -0.08146437273923739
        } else {
            var3 <- 0.14244886188108738
        }
    } else {
        if ((input[3]) > (1.4500000000000002)) {
            var3 <- -0.0950888159264695
        } else {
            var3 <- -0.09438233722389686
        }
    }
    if ((input[4]) > (1.6500000000000001)) {
        if ((input[3]) > (5.3500000000000005)) {
            var4 <- -0.8824095771015287
        } else {
            var4 <- -0.9121126703829481
        }
    } else {
        if ((input[3]) > (4.450000000000001)) {
            var4 <- -1.1277829563828181
        } else {
            var4 <- -1.1794405099157212
        }
    }
    if ((input[3]) > (4.750000000000001)) {
        if ((input[3]) > (5.150000000000001)) {
            var5 <- 0.16625543464258166
        } else {
            var5 <- 0.09608601737074281
        }
    } else {
        if ((input[1]) > (4.950000000000001)) {
            var5 <- -0.09644547407948921
        } else {
            var5 <- -0.08181864271444342
        }
    }
    return(softmax(c((var0) + (var1), (var2) + (var3), (var4) + (var5))))
}


# Filename: 	code.r
# Project: 	/Users/shume/Developer/stat/IntroductoryStatistics/01
# Author: 	shumez <https://github.com/shumez>
# Created: 	2019-05-25 14:06:7
# Modified: 	2019-05-25 14:21:43
# -----
# Copyright (c) 2019 shumez

# 01

require(ggplot2)
install.packages("dplyr")
require(dplyr)
library(dplyr)
install.packages("tidyr")
library(tidyr)

dir = '~/Developer/stat/IntroductoryStatistics/01'
setwd(dir)
dir_fig = file.path(dir, 'fig')

# Collaborative Exercise

dat = c(5, 5.5, 6, 6, 6, 6.5, 6.5, 6.5, 6.5, 7, 7, 8, 8, 9)
df = data.frame(SleepingTime=dat)
View(df)

p <- ggplot(df, aes(SleepingTime))
p <- p + geom_dotplot(binwidth = .5, dotsize=1)#, method = "histodot")
p %+% geom_histogram(binwidth = .5)
p <- p + labs(x='Sleeping Time')
ggsave(file.path(dir_fig,'0102.png'), width = 3, height = 3)

# 01.01.

# 01.02.
College = c('De Anza College', 'Foothill College')
FullTime = c(9200, 4059)
PartTime = c(13296, 10124)
df = data.frame(College=College, 'FullTime'=FullTime, 'PartTime'=PartTime)
View(df)

dat <- gather(df, 'category', 'number', -College)
dat$category[which(dat$category=='FullTime')] <- 'Full-time'
dat$category[which(dat$category=='PartTime')] <- 'Part-time'
dat$College <- as.character(dat$College)
dat$College[which(dat$College=='De Anza College')] <- 'De Anza'
dat$College[which(dat$College=='Foothill College')] <- 'Foothill'
View(dat)
ggplot(dat, aes(College, number, fill=category)) + geom_col()
ggsave(file.path(dir_fig, '0106.png'), width = 3, height = 3)

# 01.02.03. Variation in Data
cans = c(15.8, 16.1, 15.2, 14.8, 15.8, 15.9, 16.0, 15.5)
sd(cans)

# 01.03. Frequency, Frequency Tables, and Levels of Measurement

# 01.03.03. Frequency

StudentWorkHours = data.frame(DataValue=c(2, 3, 4, 5, 6, 7), Frequency=c(3, 5, 3, 6, 2, 1))
StudentWorkHours$RelativeFrequency = StudentWorkHours$Frequency/sum(StudentWorkHours$Frequency)
StudentWorkHours$CumulativeRelativeFrequency = rep(0, length(StudentWorkHours$RelativeFrequency))
for (i in 1:length(StudentWorkHours$RelativeFrequency)) {
  if(i == 1){
    StudentWorkHours$CumulativeRelativeFrequency[1] = StudentWorkHours$RelativeFrequency[1]
  }else{
    StudentWorkHours$CumulativeRelativeFrequency[i] = StudentWorkHours$RelativeFrequency[i] + StudentWorkHours$CumulativeRelativeFrequency[i-1]
  }
}
View(StudentWorkHours)

SoccerPlayerHeight = data.frame(height=c('59.95–61.95', '61.95–63.95', '63.95–65.95', '65.95–67.95', '67.95–69.95', '69.95–71.95', '71.95–73.95', '73.95–75.95'),
                                height_from = c(59.95, 61.95, 63.95, 65.95, 67.95, 69.95, 71.95, 73.95),
                                height_to = c(61.95, 63.95, 65.95, 67.95, 69.95, 71.95, 73.95, 75.95),
                                frequency=c(5, 3, 15, 40, 17, 12, 7, 1))
SoccerPlayerHeight$RelativeFrequency = SoccerPlayerHeight$frequency / sum(SoccerPlayerHeight$frequency)
View(SoccerPlayerHeight)


#' Calculate Information Value for character or factor columns
#'
#' This function calculates information value for character or factor columns of data frame.
#'
#' @param df data frame with at least two columns (predictor x and outcome y)
#' @param x column (character or factor) for which Information Value should be calculated
#' @param y column (integer) with binary outcome. y has to be a column in df data frame. It is suggested that y is factor with two levels "bad" and "good" If there are no levels good/bad than the following assumptions are applied - if y is integer, than 0=good and 1=bad. If y is factor than level 2 is assumed to mean bad and 1 good.
#' @param verbose Prints additional details when TRUE. Useful mainly for debugging.
#' @export
#' @examples
#' iv.str(german_data,"purpose","gb")
#' iv.str(german_data,"savings","gb")

iv.str <- function(df,x,y,verbose=FALSE) {
  if (!(class(df)=="data.frame")) {
    stop("Parameter df has to be a data frame.")
  }
  if (!(is.character(df[, x]) || is.factor(df[, x]))) {
    stop(paste("Input is not a character or factor! Variable:", x))
    }
  if (!(is.numeric(df[, y]) || is.factor(df[, y]))) {
    stop("Outcome is not a number nor factor!")
  }
  if (length(unique(df[, y])) != 2) {
    if(verbose) paste(cat(unique(df[,y])),"\n")
    stop("Not a binary outcome")
    }
  if (!(all(sort(unique(df[, y])) == c(0,1))) && is.numeric(df[,y])) {
    stop("Numeric outcome has to be encoded as 0 (good) and 1 (bad). \n")
  }
  if (is.factor(df[,y]) && all(levels(df[,y])[order(levels(df[,y]))]==c("bad","good"))) {
    if (verbose) cat("Assuming good = level 'good' and bad = level 'bad' \n")
    total_1 <- sum(df[,y]=="bad")
    } else if (is.factor(df[,y])) {
    if (verbose) cat("Factor: Assuming bad = level 2 and good = level 1 \n")
    total_1 <- sum(as.integer(df[, y])-1)

    } else {
    if (verbose) cat("Numeric: Assuming bad = 1 and good = 0 \n")
    total_1 <-sum(df[, y])

  }

  outcome_0 <- outcome_1 <- NULL # This is needed to avoid NOTES about not visible binding from R CMD check

  total_0 <- nrow(df) - total_1
  iv_data <- data.frame(unclass(table(df[, x],df[, y])))

  if (all(names(iv_data)==c("bad","good"))) {
    iv_data <- iv_data[,c(2,1)]
  }

  names(iv_data) <- c("outcome_0","outcome_1")
  iv_data <-  within(iv_data, {
                class <- row.names(iv_data)
                variable <- x
                pct_0 <- outcome_0 / total_0
                pct_1 <- outcome_1 / total_1
                odds <-  pct_0 / pct_1
                woe <- log(odds)
                miv <- (pct_0 - pct_1) * woe
  })

  if(is.factor(df[,x])) {
    iv_data$class <- factor(iv_data$class,levels=levels(df[,x]))
  }

  iv_data <- iv_data[c("variable","class","outcome_0","outcome_1","pct_0","pct_1","odds","woe","miv")]

  if(any(iv_data$outcome_0 == 0) | any(iv_data$outcome_1 == 0)) {
    warning("Some group for outcome 0 has zero count. This will result in -Inf or Inf WOE. Replacing - ODDS=1, WoE=0, MIV=0. \n The bin is either too small or suspiciously predictive. \n You should fix this before running any model. It does not make any sense to keep WoE = 0 for such bin.")
      iv_data$woe <- ifelse(is.infinite(iv_data$woe),0,iv_data$woe)
      iv_data$miv <- ifelse(is.infinite(iv_data$miv),0,iv_data$miv)
      iv_data$odds <-ifelse(is.infinite(iv_data$odds),1,iv_data$odds)
  }

  rownames(iv_data) <- NULL
  cat (paste("Information Value",round(sum(iv_data$miv),2),"\n"))
  iv_data
}


#Analyses for Tutorial 06: Ancestral State Estimation in R
  #Parts of this tutorial are based on that of L. Revell: http://www.phytools.org/eqg2015/asr.html

library(phytools)
library(geiger)

######Discrete Anc. State Estimation
## Read data & tree
tree<-read.tree("TutorialData/anole.gp.tre",tree.names=T)
group<-read.csv('TutorialData/anole.gp.csv', row.names=1, header=TRUE,colClasses=c('factor'))
gp<-as.factor(t(group)); names(gp)<-row.names(group)

#Plot
plotTree(tree,type="fan",fsize=0.8)
cols<-setNames(palette()[1:length(unique(gp))],sort(unique(gp)))
tiplabels(pie=model.matrix(~gp-1),piecol=cols,cex=0.3)
add.simmap.legend(colors=cols,prompt=FALSE,x=0.9*par()$usr[1],
                  y=-max(nodeHeights(tree)),fsize=0.8)

### A:Parsimony estimation
     #programming NOTE: 'MPR' in APE requires unrooted tree, outgroup specification & integer states
#gp.int<-as.numeric(gp); names(gp.int)<-names(gp)
#anc.mp<-MPR(gp.int,unroot(tree), outgroup = "Anolis_occultus")

#Plot
#anc.mp.mat<-(model.matrix(~as.factor(anc.mp[,1])-1) +model.matrix(~as.factor(anc.mp[,2])-1)) /2 #matrix of states for plotting nodes
#plotTree(tree, type="fan",fsize=0.8,ftype="i")
#cols<-setNames(palette()[1:length(unique(gp.int))],sort(unique(gp.int)))
#tiplabels(pie=model.matrix(~as.factor(gp.int)-1),piecol=cols,cex=0.3)
#  NOTE: barks on nodelabels
#nodelabels(node=1:tree$Nnode+Ntip(tree),
#           pie=anc.mp.mat,piecol=cols,cex=0.3)
#add.simmap.legend(colors=cols,prompt=FALSE,x=0.9*par()$usr[1],
#                  y=-max(nodeHeights(tree)),fsize=0.8)

### B: ML estimation
anc.ML<-ace(gp,tree,model="ER",type="discrete")
  round(anc.ML$lik.anc[1:10,],3)  #show result:
#PLOT
plotTree(tree,type="fan",fsize=0.8,ftype="i")
cols<-setNames(palette()[1:length(unique(gp))],sort(unique(gp)))
nodelabels(node=1:tree$Nnode+Ntip(tree),
           pie=anc.ML$lik.anc,piecol=cols,cex=0.3)
tiplabels(pie=model.matrix(~gp-1),piecol=cols,cex=0.3)
add.simmap.legend(colors=cols,prompt=FALSE,x=0.9*par()$usr[1],
                  y=-max(nodeHeights(tree)),fsize=0.8)

#MCMC Stochastic Character Mapping (SIMMAP) based on Huelsenbeck et al. 2003
# simulate single stochastic character map using empirical Bayes method
tree.smp1<-make.simmap(tree,gp,model="ER")
plot(tree.smp1,cols,type="fan",fsize=0.8,ftype="i")  #one run.  Not overly useful.

#Must do many times
tree.smp<-make.simmap(tree,gp,model="ER",nsim=100)
anc.smp<-summary(tree.smp,plot=FALSE)
plot(anc.smp, type="fan", fsize=0.8,ftype="i")

rm(list = ls())

######Continuous Anc. State Estimation
#Data from Mahler et al. 2010. Evolution
tree<-read.tree("TutorialData/anole.svl.tre",tree.names=T)
svl<-read.csv('TutorialData/anole.svl.csv', row.names=1, header=TRUE)
svl<-as.matrix(svl)[,1]  #change to vector
anc.cont.ML<-fastAnc(tree,svl,vars=TRUE,CI=TRUE)
  #anc.cont.ML #output: estimate and 95% CI
anc.cont.ML$ace  #ancestral estimates
plot(tree)
nodelabels()

#PLOT as color map
tree.col<-contMap(tree,svl,plot=FALSE)  #runs Anc. St. Est. on branches of tree
plot(tree.col,type="fan",legend=0.7*max(nodeHeights(tree)),
     fsize=c(0.7,0.9))

#using 'ace' function in APE
anc.cont.ML2<-ace(x=svl,phy=tree, type="continuous", method="ML")
anc.cont.ML2$ace  #with APE
anc.cont.ML$ace   #with phytools: the same estimates

anc.cont.gls<-ace(x=svl,phy=tree, corStruct = corBrownian(1, tree), method="GLS")  #same as ML  (see Schluter et al. 1997)
anc.cont.gls$ace   #GLS: SCP. the same

rm(list = ls())
######################## Simulation with known nodes
tree<-pbtree(n=100,scale=1)
## simulate data with a trend
x<-fastBM(tree,internal=TRUE,mu=3)
phenogram(tree,x,ftype="off")

#estimate with no prior
a<-x[as.character(1:tree$Nnode+Ntip(tree))]
x<-x[tree$tip.label]
## let's see how bad we do if we ignore the trend
plot(a,fastAnc(tree,x),xlab="true values",
     ylab="estimated states under BM")
lines(range(c(x,a)),range(c(x,a)),lty="dashed",col="red")
title("estimated without prior information")

## incorporate prior knowledge
pm<-setNames(c(1000,rep(0,tree$Nnode)),
             c("sig2",1:tree$Nnode+length(tree$tip.label)))
## the root & two randomly chosen nodes
nn<-as.character(c(length(tree$tip.label)+1,
                   sample(2:tree$Nnode+length(tree$tip.label),2)))
pm[nn]<-a[as.character(nn)]
## prior variance
pv<-setNames(c(1000^2,rep(1000,length(pm)-1)),names(pm))
pv[as.character(nn)]<-1e-100
## run MCMC
mcmc<-anc.Bayes(tree,x,ngen=100000,
                control=list(pr.mean=pm,pr.var=pv,
                             a=pm[as.character(length(tree$tip.label)+1)],
                             y=pm[as.character(2:tree$Nnode+length(tree$tip.label))]))

anc.est<-colMeans(mcmc$mcmc[201:1001,as.character(1:tree$Nnode+length(tree$tip.label))])

plot(a,anc.est,xlab="true values",
     ylab="estimated states using informative prior")
lines(range(c(x,a)),range(c(x,a)),lty="dashed",col="red")
title("estimated using informative prior")


#author: Thomas Coleman
# Difference in Differences Regressions - Error Analysis and Graphs for data from Snow 1855

# See "Causality in the Time of Cholera" working paper at https://papers.ssrn.com/abstract=3262234
# and my John Snow project website, http://www.hilerun.org/econ/papers/snow

# This code is licensed under the BSD 2-Clause License, https://opensource.org/licenses/BSD-2-Clause

# This collect together some simple functions that produce a standard format graph, and are 'source'd
# into various notebooks

# **preperrdata** Before graphing, to prepare the data

#  * Takes in *fittedmodel* - a regression that has been already run. From this it extracts the necessary
#  parameters. Also *single*, a string which for "single" says that there is a single treatment effect.
#  * Calculates the 1849 and 1854 predicted counts and rates
#  * Calculates *approximate* 95% error bars around the predicted rates, based on whether the fitted model
#  is Poisson or Negative Binomial
#  * Produces an adjusted 1854 predicted rate, adjusting for the 1854 time effect and treatment effect, so
#  that it is comparable to the 1849 predicted rate (for purposes of plotting with error bars)

#  This function changes global data (the x1849 & x1854 dataframes) using the "<<-" instead of "<-"
#  assignment. This is poor programming style but I could not find another easy way of doing what I wanted.

# **plot2_worker** Plots actual vs predicted, with error bars around the predicted

#  * Actually does the plotting, given all the data as input arguments (sequence no. for sub-districts;
#  the actual mean or rate; predicted; the 2.5% and 97.5% points; title)
#  * btw, the hack for plotting error bars is from
#  https://stackoverflow.com/questions/13032777/scatter-plot-with-error-bars

# **plot2_worker** Is a cover function which unpacks the actual versus predicted mean from the appropriate dataframe

# **plot3** Plots actual 1849, 1854 (adjusted for time & treatment effects), predicted, with error bars

# **plotcomp** Plots actual 1849 versus 1854, with error bars around actual 1849

# **ploterrbars** is NOT a function you should use - I use it to print out .pdf versions of the graphs I want to use

preperrdata <- function(fittedmodel,single = "single") {       # This is not a good way to do this because I am
	                                                  # changing globals from within the function (using <<-
	                                                  # instead of <-)
# Function to prepare the data (error bars) for graphing
# The 2.5% and 97.5% confidence bands are calculated assuming either Poisson or Negative Binomial
# The rates are calculated by generating the counts up and down from the "expected" (from the fitted model)

	expected <- exp(predict(fittedmodel))       # expected values
	theta <- fittedmodel$theta
	x1849$predcount <<- expected[1:28]
	x1854$predcount <<- expected[29:56]
	x1849$predrate <<- 10000 * expected[1:28] / x1849$pop1851
	x1854$predrate <<- 10000 * expected[29:56] / x1854$pop1851
	xfamily <- family(fittedmodel)$family
	if (xfamily == "poisson") {         # Get 95% confidence bands depending on model used (Poiss vs Neg Binom)
		x1849$limdn <<- 10000 * qpois(.025,lambda=x1849$predcount) / x1849$pop1851
		x1849$limup <<- 10000 * qpois(.975,lambda=x1849$predcount) / x1849$pop1851
	} else {
		x1849$limdn <<- 10000 * qnbinom(.025,size=theta,mu=x1849$predcount) / x1849$pop1851
		x1849$limup <<- 10000 * qnbinom(.975,size=theta,mu=x1849$predcount) / x1849$pop1851
		x1854$limdn <<- 10000 * qnbinom(.025,size=theta,mu=x1854$predcount) / x1849$pop1851
		x1854$limup <<- 10000 * qnbinom(.975,size=theta,mu=x1854$predcount) / x1849$pop1851
		x1849$limdnact <<- 10000 * qnbinom(.025,size=theta,mu=x1849$deaths) / x1849$pop1851
		x1849$limupact <<- 10000 * qnbinom(.975,size=theta,mu=x1849$deaths) / x1849$pop1851
	}

	# Now adjust the 1854 predicted counts (and rates) for the time and treatment fixed effects -
	# effectively netting out the 1854 effect and making it 1849-equivalent
	xyr1854 <- (coef(fittedmodel)["year1854"])
	x3 <- exp(-xyr1854)
	# Adjust for the year effect only - this should make 1849 & 1854 comparable net of time effect
	x1854$rateadjyr <<- 10000 * (x1854$deaths*x3) / x1854$pop1851
	# Adjust the 1854 actual for the estimated Treatment Effect and the estimate Year Effect
	if (single == "single")  {   # model with single treatment effect
		xdegless <- (coef(fittedmodel)["supplierSouthwarkVauxhall_Lambeth:year1854"])
		xdegmore <- xdegless               # Make both treatment effects the same
		x3 <- x3 * exp(-(x1854$lambethdegree == "less_Lambeth")*xdegless)
		x3 <- x3 * exp(-(x1854$lambethdegree == "more_Lambeth")*xdegmore)
	} else if (single == "two") {      # model with two treatment effects
		xdegless <- coef(fittedmodel)["lambethdegreeless_Lambeth:year1854"]
		xdegmore <- coef(fittedmodel)["lambethdegreemore_Lambeth:year1854"]
		x3 <- x3 * exp(-(x1854$lambethdegree == "less_Lambeth")*xdegless)
		x3 <- x3 * exp(-(x1854$lambethdegree == "more_Lambeth")*xdegmore)
	} else { # model with continuous treatment (population proportions)
		xlambethperc54 <- coef(fittedmodel)["lambethperc54"]
		x3 <- x3 * exp(-(x1854$lambethperc54 * xlambethperc54))
	}

	# Adjust for year & treatment effect - so that we can compare the actuals against each other
	x1854$rateadj <<- 10000 * (x1854$deaths*x3) / x1854$pop1851

	return(xfamily)
}

# "Worker" function to plot mean, predicted, and error bars
plot2_worker <- function(yseq, xmean,xlimdn,xpred,xlimup,title) {
	xplot <- plot(xmean, yseq,
	    xlim=range(c(xmean, xpred,xlimdn,xlimup)),
	    ylim=rev(range(yseq)), col="red",
	    main=title,xlab="Mortality rate actual (red filled) vs predicted (empty circle)",ylab="sub-district",
	    pch=19)
	lines(xpred, yseq, type="p",
	    xlim=range(c(xmean, xpred,xlimdn,xlimup)),
	    ylim=rev(range(yseq)),
	    pch=1)
	# horizontal error bars
	xplot <- arrows(xlimdn, yseq, xlimup, yseq, length=0.05, angle=90, code=3,lty=3)
	xplot
}
# "Cover" function that takes in the dataframe and unpacks
plot2 <- function(confidata,xsupplier,title) {
	xmean <- subset(confidata,supplier==xsupplier)[,"rate"]
	xlimdn <- subset(confidata,supplier==xsupplier)[,"limdn"]
	xlimup <- subset(confidata,supplier==xsupplier)[,"limup"]
	xpred <- subset(confidata,supplier==xsupplier)[,"predrate"]
	yseq <- subset(confidata,supplier==xsupplier)[,"seq"]
#	xtitle <- paste(xsupplier,title)
	xplot <- plot2_worker(yseq,xmean,xlimdn,xpred,xlimup,title)
}

plot3 <- function(confidata1849,confidata1854,xsupplier,title) {
	xmean <- subset(confidata1849,supplier==xsupplier)[,"rate"]
	xlimdn <- subset(confidata1849,supplier==xsupplier)[,"limdn"]
	xlimup <- subset(confidata1849,supplier==xsupplier)[,"limup"]
	xpred <- subset(confidata1849,supplier==xsupplier)[,"predrate"]
	y <- subset(confidata1849,supplier==xsupplier)[,"seq"]
	x1854adj <- subset(confidata1854,supplier==xsupplier)[,"rateadj"]
	xplot <- plot(xmean, y,
	    xlim=range(c(xmean, xpred,xlimdn,xlimup,x1854adj)),
	    ylim=rev(range(y)), col="red",
	    main=title,xlab="Mortality: 1849 red circle, 1854 blue diamond vs predicted (circle)",ylab="sub-district",
	    pch=19)
	lines(xpred, y, type="p",
	    xlim=range(c(xmean, xpred,xlimdn,xlimup)),
	    ylim=rev(range(y)),
	    pch=1)
	lines(x1854adj, y, type="p",
	    xlim=range(c(xmean, xpred,xlimdn,xlimup)),
	    ylim=rev(range(y)), col="blue",
	    pch=17)

	# horizontal error bars
	xplot <- arrows(xlimdn, y, xlimup, y, length=0.05, angle=90, code=3,lty=3)
}

# Plotting joint region for 1849 & 1854 with error bars for each
plotcomp <- function(confidata1849,confidata1854,xsupplier,title) {
	xmean <- subset(confidata1849,supplier==xsupplier)[,"rate"]
	xlimdn <- subset(confidata1849,supplier==xsupplier)[,"limdnact"]
	xlimup <- subset(confidata1849,supplier==xsupplier)[,"limupact"]
	xpred <- subset(confidata1849,supplier==xsupplier)[,"predrate"]
	y <- subset(confidata1849,supplier==xsupplier)[,"seq"]
	xmean1854 <- subset(confidata1854,supplier==xsupplier)[,"rateadjyr"]
#	xlimdn1854 <- subset(confidata1854,supplier==xsupplier)[,"limdn"]
#	xlimup1854 <- subset(confidata1854,supplier==xsupplier)[,"limup"]
	xplot <- plot(xmean, y,
	    xlim=range(c(xmean, xmean1854,xlimdn,xlimup)),
	    ylim=rev(range(y)), col="red",
	    main=title,xlab="Mortality: 1849 red circle, 1854 blue diamond",ylab="sub-district",
	    pch=19)
	lines(xmean1854, y, type="p",
	    xlim=range(c(xmean, xmean1854,xlimdn,xlimup)),
	    ylim=rev(range(y)), col="blue",
	    pch=17)

	# horizontal error bars
	xplot <- arrows(xlimdn, y, xlimup, y, length=0.05, angle=90, code=3,,lty=3)
#	xplot <- arrows(xlimdn1854, y, xlimup1854, y, length=0.05, angle=90, code=3)
}

ploterrbars <- function(fittedmodel,plotname,single = "single") {       # This is not a good way to do this because I am
	                                                  # changing globals from within the function (using <<-
	                                                  # instead of <-)
	xfamily <- preperrdata(fittedmodel,single)  # this function modifies global data

	pdf(paste("../paper/figures/errbar_",plotname,"a.pdf",sep=""))
		plot2(x1849,"SouthwarkVauxhall",paste("First-12 Southwark-only ",xfamily," 1849 "))
	dev.off()
	pdf(paste("../paper/figures/errbar_",plotname,"b.pdf",sep=""))
		plot2(x1849,"SouthwarkVauxhall_Lambeth",paste("Next-16 Jointly-Supplied ",xfamily," 1849 "))
	dev.off()
	pdf(paste("../paper/figures/errbar_",plotname,"c.pdf",sep=""))
		plot3(x1849,x1854,"SouthwarkVauxhall",paste("First-12 Southwark-only ",xfamily," 1849vs1854 "))
	dev.off()
	pdf(paste("../paper/figures/errbar_",plotname,"d.pdf",sep=""))
		plot3(x1849,x1854,"SouthwarkVauxhall_Lambeth",paste("Next-16 Jointly-Supplied ",xfamily," 1849vs1854 "))
	dev.off()
	if (xfamily != "poisson") {         # Plot comparison for joint region only for Negative Binomial
		pdf(paste("../paper/figures/errbar_",plotname,"e.pdf",sep=""))
			plotcomp(x1849,x1854,"SouthwarkVauxhall",paste("First-12 Southwark-only ",xfamily," 1849vs1854 "))
		dev.off()
		pdf(paste("../paper/figures/errbar_",plotname,"f.pdf",sep=""))
			plotcomp(x1849,x1854,"SouthwarkVauxhall_Lambeth",paste("Next-16 Jointly-Supplied ",xfamily," 1849vs1854 "))
		dev.off()
	}

}


if (latitude < 0):
    if (sowingDay > sDsa_sh):
        fixPhyll = p * (1 - rp * min(sowingDay - sDsa_sh, sDws))
    else: fixPhyll = p
else:
    if (sowingDay < sDsa_nh):
        fixPhyll = p * (1 - rp * min(sowingDay, sDws))
    else: fixPhyll = p


params <- chrpfdnm %>% group_by(ID) %>% summarise(n=n()) %>% summarise(mean=mean(n), sd=sd(n))

##############################################################################
# Calculate individual-level AUC from the GoNL DNMs
#
# Each iteration calculates the AUC for the following:
# AUC under 3-mer model
# Permuted AUC under 3-mer model
# AUC under logit model
# Permuted AUC under logit model
##############################################################################
nperm <- 258

aucperm <- rep(0,nperm)
aucperm3 <- aucperm

ids <- unique(chrpfdnm$ID)
numind <- length(ids)
aucind <- rep(0, numind)
aucind3 <- aucind

for(i in 1:nperm){
	### Run permutations
	cat("Permuting AUC", "(", i, "of", nperm, ")...\n")
	ndnms <- round(rnorm(1, params$mean, params$sd), 0)
	nsamp <- 1000000

	chrpsub <- rbind(chrpfdnm[sample(nrow(chrpfdnm), ndnms),], chrpfa) %>%
		arrange(MU)
	chrpsub$prop <- cumsum(chrpsub$OBS)/sum(chrpsub$OBS)
	chrpsub2 <- chrpsub[sample(nrow(chrpsub), nsamp),] %>%
	  arrange(MU) %>%
	  mutate(ntile=ntile(MU, 1000))

	chrpsub3 <- chrpsub %>% arrange(MU3)
	chrpsub3$prop <- cumsum(chrpsub3$OBS)/sum(chrpsub3$OBS)
	chrpsub3a <- chrpsub3[sample(nrow(chrpsub3), nsamp),] %>%
		arrange(MU3, prop) %>%
		mutate(ntile=ntile(MU3, 1000))

	auctmp <- chrpsub2 %>% summarise(AUC=1-sum(prop)/nsamp)
	aucperm[i] <- auctmp$AUC
	auctmp3 <- chrpsub3a %>% summarise(AUC=1-sum(prop)/nsamp)
	aucperm3[i] <- auctmp3$AUC

	curid<-ids[i]
	### Get empirical AUC
	cat("Calculating AUC for", curid, "(", i, "of", numind, ")...\n")
	datid<-chrpfdnm %>% filter(ID==curid)

	tmpdat<-rbind(datid, chrpfa) %>% arrange(MU)
	tmpdat$prop<-cumsum(tmpdat$OBS)/sum(tmpdat$OBS)
	tmpdat2<-tmpdat[sample(nrow(tmpdat), nsamp),] %>%
	  arrange(MU) %>%
	  mutate(ntile=ntile(MU, 1000))

	### Repeat with 3-mer model
	tmpdat3<-tmpdat %>% arrange(MU3)
	tmpdat3$prop<-cumsum(tmpdat3$OBS)/sum(tmpdat3$OBS)
	tmpdat3s<-tmpdat3[sample(nrow(tmpdat3), nsamp),] %>%
		arrange(MU3, prop) %>%
		mutate(ntile=ntile(MU3, 1000))

	tmpauc <- tmpdat2 %>% summarise(AUC=1-sum(prop)/nsamp)
	aucind[i]<-tmpauc$AUC
	tmpauc3 <- tmpdat3s %>% summarise(AUC=1-sum(prop)/nsamp)
	aucind3[i]<-tmpauc3$AUC
}

##############################################################################
# Calculate AUC for each individual under the logit and 3-mer models
##############################################################################
# ids<-unique(chrpfdnm$ID)
# numind<-length(ids)
# aucind<-rep(0, numind)
# aucind3<-aucind
#
# for(i in 1:numind){
# 	curid<-ids[i]
# 	cat("Calculating AUC for", curid, "(", i, "of", numind, ")...\n")
# 	datid<-chrpfdnm %>% filter(ID==curid)
# 	tmpdat<-rbind(datid, chrpfa) %>% arrange(MU)
# 	tmpdat$prop<-cumsum(tmpdat$OBS)/sum(tmpdat$OBS)
#
# 	nsamp<-50000
#
# 	tmpdat2<-tmpdat[sample(nrow(tmpdat), nsamp),] %>%
# 	  arrange(MU) %>%
# 	  mutate(ntile=ntile(MU, 1000))
#
# 	tmpauc <- tmpdat2 %>% summarise(AUC=1-sum(prop)/nsamp)
# 	aucind[i]<-tmpauc$AUC
#
# 	### Repeat with 3-mer model
# 	tmpdat3<-tmpdat %>% arrange(MU3)
# 	tmpdat3$prop<-cumsum(tmpdat3$OBS)/sum(tmpdat3$OBS)
#
# 	tmpdat3s<-tmpdat3[sample(nrow(tmpdat3), nsamp),] %>%
# 		arrange(MU3, prop) %>%
# 		mutate(ntile=ntile(MU3, 1000))
#
# 	tmpauc3 <- tmpdat3s %>% summarise(AUC=1-sum(prop)/nsamp)
# 	aucind3[i]<-tmpauc3$AUC
# }

##############################################################################
# Function checks if elements in a exist in b
# Output is binary vector of length same as b
##############################################################################
toBin <- function(a,b){
	as.numeric(is.element(b,a))
}

##############################################################################
# Simulate mutations by randomly selecting sites and checking if
# Bernoulli(mu)==1; loop continues until we have simulated the same number
# of sites as in observed data
##############################################################################
nsim<-200

for(i in 1:nsim){
	cat("Running simulation", i, "of", nsim, "...\n")
	nsample <- 20000
	mutated <- c()
	# nsites<-round(rnorm(1, params$mean, params$sd), 0)
  nsites <- 10171

	while(length(mutated) < nsites){
		rowind <- sample(nrow(chrp), nsample)
		row <- chrp[rowind,]
		mu <- row$MU

		batch <- sapply(mu, function(x) rbinom(1,1,x))
		mutated <- c(mutated, row$POS[which(as.logical(batch))])
	}

	# chrp$TMP <- toBin(mutated[1:nsites], chrp$POS)
	mus <- toBin(mutated, chrp$POS)
	nsimi <- sum(mus)
	chrp$last <- cumsum(mus)/nsimi

	colnames(chrp)[ncol(chrp)] <- paste0("simprop", i)
}

##############################################################################
# Subsample 100k sites, resort by mu, add percentile column, and coerce
# from wide to long format
#
# Can delete chrp after this step
##############################################################################
cat("Subsampling data...\n")
nsamp<-100000

chrp2<-chrp[sample(nrow(chrp), nsamp),] %>%
  arrange(MU) %>%
  mutate(ntile=ntile(MU, 1000)) %>%
  gather(group, val, .dots=grep("prop", names(chrp)))

auc <- chrp2 %>% group_by(group) %>% summarise(AUC=1-sum(val)/nsamp)

cat("Cleaning up memory...\n")
gc()

##############################################################################
# Summarize simulated data for ROC plots
##############################################################################
chrp2_nt_mean<-chrp2 %>%
  group_by(group, ntile) %>%
  summarise(val=mean(val))

# Get subsets of true and simulated data
chrp2_nt_sim<-chrp2_nt_mean[chrp2_nt_mean$group!="prop",]
chrp2_nt_obs<-chrp2_nt_mean[chrp2_nt_mean$group=="prop",]

# Calculate lower/upper range from simulations
# (can switch commented summarise() lines to use CI)
chrp2_lb<-chrp2_nt_sim %>%
  group_by(ntile) %>%
  # summarise(lb=t.test(val)$conf.int[1]) %>%
	summarise(lb=max(val)) %>%
  mutate(group="lb") %>%
  dplyr::select(group, ntile, lb)

chrp2_ub<-chrp2_nt_sim %>%
  group_by(ntile) %>%
  # summarise(ub=t.test(val)$conf.int[2]) %>%
	summarise(ub=min(val)) %>%
  mutate(group="ub") %>%
  dplyr::select(group, ntile, ub)

# Add bounds to data frame
chrp2_nt_obs$lb <- chrp2_lb$lb
chrp2_nt_obs$ub <- chrp2_ub$ub

##############################################################################
# Combine AUC data to generate ROC plot
##############################################################################
full_auc_dat<-rbind(data.frame(chrp2_nt_obs[,1:3]), data.frame(chrp3b))
names(full_auc_dat)[1]<-"Model"
full_auc_dat$Model<-ifelse(full_auc_dat$Model=="prop", "Logit", "3-mer")
full_auc_bounds<-chrp2_nt_obs[,c(2:5)]

ggplot()+
  geom_line(data=full_auc_dat,
		aes(x=(1000-ntile)/1000, y=1-val, group=Model, colour=Model), size=1.2)+
  geom_abline(intercept=0, slope=1)+
  geom_ribbon(data=full_auc_bounds,
		aes(x=(1000-ntile)/1000, y=1-val, ymin=1-ub, ymax=1-lb), alpha=0.2)+
	scale_colour_manual(values=cbbPalette[2:3])+
  coord_cartesian(xlim=c(0,1))+
	xlab("False Positive Rate")+
	ylab("True Positive Rate")+
  theme_bw()+
	theme(
		# legend.position="none",
		axis.title.x=element_text(size=20),
		axis.title.y=element_text(size=20))
  # scale_y_continuous(limits=c(0,1))+
  # scale_x_continuous(limits=c(0,1000), labels=c())
ggsave("/net/bipolar/jedidiah/mutation/images/pseudo_roc_curves.png", height=4, width=7.25)

##############################################################################
# Get AUC data for distribution plots
##############################################################################
auc$model<-"simulated"
auc$obs<-"simulated"

# aucind_df<-data.frame(AUC=aucind, gp="ind")
aucsim<-auc[-1,2:4]
auclogitobs<-data.frame(AUC=aucind, model="logit", obs="observed")
auc3merobs<-data.frame(AUC=aucind3, model="3-mer", obs="observed")
auclogitperm<-data.frame(AUC=aucperm, model="logit", obs="permuted")
auc3merperm<-data.frame(AUC=aucperm3, model="3-mer", obs="permuted")

# All AUCs
plotaucfull<-rbind(aucsim, auclogitobs, auc3merobs, auclogitperm, auc3merperm)
plotaucfull$model<-as.factor(plotaucfull$model)
plotaucfull$model<-relevel(plotaucfull$model, "3-mer")

# No simulations
plotaucnosim<-rbind(auclogitobs, auc3merobs, auclogitperm, auc3merperm)
plotaucnosim$model<-as.factor(plotaucnosim$model)
plotaucnosim$model<-relevel(plotaucnosim$model, "3-mer")

# Only observed data (no simulation or permutations)
plotaucobsonly<-rbind(auclogitobs, auc3merobs)
plotaucobsonly$model<-as.factor(plotaucobsonly$model)
plotaucobsonly$model<-relevel(plotaucobsonly$model, "3-mer")

# Observed + Simulated (no permutations)
plotaucnoperm<-rbind(aucsim, auclogitobs, auc3merobs)
plotaucnoperm$model<-as.factor(plotaucnoperm$model)
plotaucnoperm$model<-relevel(plotaucnoperm$model, "3-mer")

plotaucmean<-plotaucobsonly %>%
	group_by(model, obs) %>%
	summarise(AUC=mean(AUC))

ggplot()+
	geom_density(data=plotaucfull, aes(x=AUC, colour=model, linetype=obs))+
	geom_density(data=aucsim, aes(x=AUC), colour="black")+
	scale_colour_manual(values=cbbPalette[c(2,3,1)])+
	# geom_vline(data=plotaucmean,
	# 	aes(xintercept=AUC, colour=model, linetype=obs))+
	theme_classic()+
	guides(linetype=F)+
	theme(legend.position="bottom")
ggsave("/net/bipolar/jedidiah/mutation/images/auc_hist.png", height=4, width=4)

# Add paternal age
ages<-read.table("/net/bipolar/jedidiah/mutation/reference_data/DNMs/gonl_fam_age.txt", header=T)
names(ages)<-c("ID", "nDNM", "FatherAge", "MotherAge", "Coverage")
plotaucobsonly<-rbind(auclogitobs, auc3merobs)
plotaucobsonly$ID<-c(ids, ids)
plotaucobsonly<-merge(plotaucobsonly, ages, by="ID")

##############################################################################
# Additional data summaries for paternal age, etc.
##############################################################################
auc_age_cor<-plotaucobsonly %>%
	group_by(model) %>%
	summarise(cor=cor(AUC, FatherAge, method="spearman"),
		cor.p=cor.test(AUC, FatherAge, method="spearman")$p.value,
		cornum=cor(AUC, nDNM, method="spearman"),
		cornum.p=cor.test(AUC, nDNM, method="spearman")$p.value)

fao<-plotaucobsonly %>%
	filter(model=="logit") %>%
	filter(AUC>0.669 | AUC<0.597) %>%
	mutate(quart=ifelse(AUC>0.669, "yes", "no")) %>%
	dplyr::select(quart, FatherAge) %>%
	spread(quart, FatherAge)

# OLD VERSION--early attempt at pseudo-ROC curves
# ggplot(chrp3, aes(x=1000-ntile, y=1-val, group=group, colour=group))+
#   geom_line()
# ggsave("/net/bipolar/jedidiah/mutation/images/psuedo_roc_chr4a.png")

# OLD VERSION--uses true ROC calculations
# devtools::install_github("hadley/ggplot2")
# devtools::install_github("sachsmc/plotROC")
# library(plotROC)
#
# basicplot2<-ggplot(chrp2, aes(d = OBS, m = MU))+
# 	geom_roc()
#
# basicplot2+annotate("text", x = .75, y = .25,
#            label = paste("AUC =", round(calc_auc(basicplot)$AUC, 2)))+
# 	style_roc()
# ggsave("/net/bipolar/jedidiah/mutation/images/chr18_roc2.png")


kasitsiTabel <-c( 0, 1, 3, 6, 10, 15, 21, 28, 36, 45, 55, 66, 78, 91, 105, 120, 136, 153, 171, 190, 210, 231, 253, 276, 300, 325, 351, 378, 406, 435, 465, 496, 528, 561, 595, 630, 666, 703, 741, 780, 820, 861, 903, 946, 990, 1035, 1081, 1128, 1176, 1225)
pihta <- c(0, 0, 2, 3, 9, 12, 21, 26, 30, 35, 41, 46, 54, 59, 67, 74, 82, 89, 97, 106, 116, 125, 133, 144, 154, 163, 173, 184, 196, 207, 219, 230, 242, 255, 267, 280, 292, 305, 319, 332, 346, 361, 375, 390, 404, 419, 435, 450, 466, 481)

#X11.options(type="nbcairo")
plot(kasitsiTabel, type="o", col="blue", ylim=c(0, 1300), xlab = "N", ylab="K")
lines(pihta, type="o", pch=22, lty=2, col="red")

leg <- c(expression("k"[0]), expression("k"[1]))
cols <- c("blue", "red")
legend(0, 1300, leg, cex = 1, cols, pch=22, lty=1:2)