だいえと - プログラム倉庫

import java.io.*;

class pro{

String[] data=new String[50000];
int datanumber;
double[] pos=new double[101];
int s,sx,sxs,t,tx,txs,slong,tlong;
String str;
int strlong,dicnumber,wordnumber;
String[][] ph=new String[500][11];
int[][] nph=new int[500][11];
String[] dic=new String[1000];
double[] p=new double[1000];
double[] fp=new double[1000];
double[] mp=new double[1000];
double[][] v=new double[500][11];
int[][] gototx=new int[500][11];
int[] m1=new int[1000];
int[] m2=new int[1000];
int[] num=new int[1000];
int[] nword=new int[500];
int m,ms,mknumber;
double v1,maxv;
String[] word=new String[500];
double c1,c2,h;
int tr;

public static void main(String args[]){
pro test=new pro();
}

pro(){

dicnumber=0;
mknumber=0;
fp[0]=1;
p[0]=1;
mp[0]=1;

makepos();

readfile("ban.txt");

for(tr=1;tr<datanumber+1;tr++){

str=data[tr];
strlong=str.length();

for(s=1;s<strlong+1;s++){
slong=s;
if(slong>10)slong=10;
for(sx=1;sx<slong+1;sx++){
ph[s][sx]=str.substring(s-sx,s);
}
}

for(s=1;s<strlong+1;s++){
slong=s;
if(slong>10)slong=10;
for(sx=1;sx<slong+1;sx++){
ms=0;
for(m=1;m<dicnumber+1;m++){
if(ph[s][sx].equals(dic[m]))ms=m;
}
nph[s][sx]=ms;
}
}

slong=strlong;
if(slong>10)slong=10;
for(s=1;s<slong+1;s++){
sx=nph[s][s];
v[s][s]=s*pos[s]*fp[sx];
}

m=mk(nph[1][1],nph[2][1]);
v[2][1]=pos[1]*mp[m]+v[1][1];
gototx[2][1]=1;

for(s=3;s<strlong+1;s++){
slong=s-1;
if(slong>10)slong=10;
for(sx=1;sx<slong+1;sx++){
t=s-sx;
tlong=t;
if(tlong>10)tlong=10;
maxv=-999;
txs=0;  
for (tx=1;tx<tlong+1;tx++){
m=mk(nph[t][tx],nph[s][sx]);
v1=sx*pos[sx]*mp[m]+v[t][tx];
if(v1>maxv)txs=tx;  
if(v1>maxv)maxv=v1;
}
gototx[s][sx]=txs;
v[s][sx]=maxv;  
}
}


s=strlong;
maxv=-999;
sxs=0;
slong=strlong;
if(slong>10)slong=10;
for(sx=1;sx<slong+1;sx++){
if(v[s][sx]>maxv)sxs=sx;
if(v[s][sx]>maxv)maxv=v[s][sx];
}

makeword(sxs);

for(s=1;s<wordnumber+1;s++){
sxs=0;
for(sx=1;sx<dicnumber+1;sx++){
if(word[s].equals(dic[sx]))sxs=sx;
}
if(sxs==0)dicnumber=dicnumber+1;
if(sxs==0)dic[dicnumber]=word[s];
if(sxs==0)nword[s]=dicnumber;
if(sxs>0)nword[s]=sxs;
}

for(s=0;s<dicnumber+1;s++){
num[s]=0;
}

for(s=1;s<wordnumber+1;s++){
m=nword[s];
num[m]=num[m]+1;
}

for(s=0;s<dicnumber+1;s++){
p[s]=0.95*p[s]+0.05*(double)num[s]/wordnumber;
}

m=nword[1];
for(s=1;s<dicnumber+1;s++){
if(s==m)fp[m]=0.95*fp[m]+0.05;
if(s==m)fp[m]=0.95*fp[m];
}



for(s=1;s<wordnumber;s++){
m=mk(nword[s],nword[s+1]);
if(m==0)mknumber=mknumber+1;
if(m==0)m1[mknumber]=nword[s];
if(m==0)m2[mknumber]=nword[s+1];
}


for(s=1;s<mknumber+1;s++){
c1=0;
c2=0;
for(sx=1;sx<wordnumber;sx++){
if(nword[sx]==s)c1=c1+1;
}
for(sx=1;sx<wordnumber;sx++){
h=0;
if(nword[sx]==s)h=h+1;
if(nword[sx+1]==s)h=h+1;
if(h==2)c2=c2+1;
}
if(c1>0)mp[s]=0.95*mp[s]+0.05*(double)c2/c1;
}
mp[0]=0.95*mp[0];

System.out.println(dicnumber);

if(dicnumber>500)diet();

}




}

void diet(){
int s,sx,s1,s2,sx1,sx2;
int[] ch=new int[1000];
int[] newm1=new int[1000];
int[] newm2=new int[1000];
String[] newdic=new String[1000];
double[] newp=new double[1000];
double[] newfp=new double[1000];
double[] newmp=new double[1000];
int newdicnumber;
double p1;
int newmknumber;
sx=0;
for(s=1;s<dicnumber+1;s++){
ch[s]=0;
if (p[s]>0.001)sx=sx+1;
if (p[s]>0.001)ch[s]=sx;
}
newdicnumber=sx;
for(s=1;s<dicnumber+1;s++){
if (ch[s]>0)sx=ch[s];
if (ch[s]>0)newdic[sx]=dic[s];
if (ch[s]>0)newp[sx]=p[s];
if (ch[s]>0)newfp[sx]=p[s];
}

p1=0;
for(s=1;s<newdicnumber+1;s++){
p1=p1+newp[s];
}

newp[0]=1-p1;
if(newp[0]<0)newp[0]=0;

p1=0;
for(s=1;s<newdicnumber+1;s++){
p1=p1+newfp[s];
}

newfp[0]=1-p1;
if(newfp[0]<0)newfp[0]=0;

sx=0;
for(s=1;s<mknumber+1;s++){
s1=m1[s];
s2=m2[s];
h=0;
if(ch[s1]>0)h=h+1;
if(ch[s2]>0)h=h+1;
if(h==2)sx=sx+1;
if(h==2)newm1[sx]=ch[s1];
if(h==2)newm2[sx]=ch[s2];
if(h==2)newmp[sx]=mp[s];
}
newmknumber=sx;

p1=0;
for(s=1;s<newmknumber+1;s++){
p1=p1+newmp[s];
}
newmp[0]=1-p1;
if(newmp[0]<0)newmp[0]=0;

dicnumber=newdicnumber;
mknumber=newmknumber;
mp=newmp;
p=newp;
fp=newfp;

}

void makeword(int sxs){

int s,sx,t,tx;
int[] op1=new int[500];
int[] op2=new int[500];  
int z,c,h;
sx=sxs;
s=strlong;  
z=s-sx;
op1[1]=s;
op2[1]=sx;
c=1;
while(z>1){
t=s-sx;
tx=gototx[s][sx];
s=t;
sx=tx;
c=c+1;
op1[c]=s;
op2[c]=sx;
z=s-sx;
}

h=0;
if(op1[c]>op2[c])h=100;
if(h>50)op1[c+1]=op1[c]-op2[c];
if(h>50)op2[c+1]=op1[c+1];
if(h>50)c=c+1;

wordnumber=c;
for(c=1;c<wordnumber+1;c++){
s=op1[c];
sx=op2[c];
word[wordnumber+1-c]=ph[s][sx];
}

}

int mk(int sx1,int sx2){
int i,sxs;
int h;
sxs=0;
for(i=1;i<mknumber+1;i++){
h=0;
if(m1[i]==sx1)h=h+1;
if(m2[i]==sx2)h=h+1;
if(h==2)sxs=i;
}
return sxs;
}


void makepos(){
double lam,x1,x2;
double[] cx=new double[101];
int k;
lam=4.8;
x1=1-lam;
cx[1]=1;
cx[2]=1;
for(k=3;k<101;k++){
cx[k]=(k-1)*cx[k-1];
}
for(k=1;k<101;k++){
x2=Math.pow(lam-1,k);
pos[k]=x2*Math.exp(x1)/cx[k];
}

}

void readfile(String file){

String st;
BufferedReader br;
s=0;

try {
br = new BufferedReader(new InputStreamReader(new FileInputStream(file),"SJIS"));  
while((st = br.readLine()) != null) {  
s=s+1;
data[s]=st;  
}
br.close();
} catch (IOException e) {System.out.println(e);}  
datanumber=s;

}




}