初めての大量データ - プログラム倉庫

import java.io.*;

class pro{

String[] data=new String[30000];
String str;
int n,s,sx,strlong,datanumber;
int[] z=new int[1000];
int[] zz=new int[1000];
int dicnumber,phnumber,wordnumber;
String[] ph=new String[500];
String[] word=new String[500];
int[] nph=new int[500];
int[] nword=new int[500];
String[] dic=new String[50000];
double[] p=new double[50000];
double[] q=new double[50000];
double point,maxpoint;
int[] mvnumber=new int[1000];
int[][] mv=new int[1000][500];
int trial,time,tm;
int[] num=new int[50000];

public static void main(String args[]){
pro test=new pro();
}

pro(){

dicnumber=0;
p[0]=1;


readfile("ban.txt");

tm=0;

for(n=1;n<datanumber+1;n++){

str=data[n];

strlong=str.length();

maxpoint=-999;
for(trial=1;trial<10;trial++){

z[0]=1;
for(s=1;s<strlong;s++){
z[s]=0;
if(Math.random()>0.5)z[s]=1;
}

zz[1]=0;
sx=1;
for(s=1;s<strlong;s++){
if(z[s]==1)sx=sx+1;
if(z[s]==1)zz[sx]=s;
}
phnumber=sx;

for(s=1;s<phnumber;s++){
ph[s]=str.substring(zz[s],zz[s+1]);
}
s=phnumber;
ph[s]=str.substring(zz[s],strlong);


for(s=1;s<phnumber+1;s++){
nph[s]=0;
for(sx=1;sx<dicnumber+1;sx++){
if(ph[s].equals(dic[sx]))nph[s]=sx;
}
}

sx=nph[1];
point=fast(sx);
for(s=1;s<phnumber;s++){
point=pr(nph[s],nph[s+1])+point;
}

if(point>maxpoint)wordnumber=phnumber;
if(point>maxpoint)word=ph;
if(point>maxpoint)nword=nph;
if(point>maxpoint)maxpoint=point;
}

for(s=1;s<wordnumber+1;s++){
if(nword[s]==0)makedic(s);
}

for(s=1;s<wordnumber+1;s++){
if(nword[s]==0)nword[s]=makenword(s);
}

for(s=1;s<dicnumber+1;s++){
num[sx]=0;
}

for(s=1;s<wordnumber+1;s++){
sx=nword[s];
num[sx]=num[sx]+1;
}

for(s=1;s<dicnumber+1;s++){
q[s]=(double)num[s]/wordnumber;
}


for(s=1;s<dicnumber+1;s++){
p[s]=0.9*p[s]+0.1*q[s];
}


tm=tm+1;
if(tm>999)tm=1;
mvnumber[tm]=wordnumber;
for(s=1;s<wordnumber+1;s++){
mv[tm][s]=nword[s];
}


System.out.println(n);
System.out.println(dicnumber);

}

}

int makenword(int s){
int sxs;
sxs=0;
for(sx=1;sx<dicnumber+1;sx++){
if(word[s].equals(dic[sx]))sxs=sx;
}
return sxs;
}

void makedic(int s){
int h;
h=0;
for(sx=1;sx<dicnumber+1;sx++){
if(word[s].equals(dic[sx]))h=100;
}
if(h<50)dicnumber=dicnumber+1;
if(h<50)dic[dicnumber]=word[s];
}

double pr(int m1,int m2){
int t1,t2;
int pr1,pr2;
int h;
double px;
px=0.01;
pr1=0;
pr2=0;
for(t1=1;t1<1000;t1++){
for(t2=1;t2<mvnumber[t1];t2++){
if(mv[t1][t2]==m1)pr1=pr1+1;
}
}

for(t1=1;t1<1000;t1++){
for(t2=1;t2<mvnumber[t1];t2++){
h=0;
if(mv[t1][t2]==m1)h=h+1;
if(mv[t1][t2+1]==m2)h=h+1;
if(h==2)pr2=pr2+1;
}
}
if(pr1>0)px=pr2/pr1;
if(px<0.01)px=0.01;
return px;
}

double fast(int m1){
int t1,pr1,pr2,h;
double px;
px=0.01;
pr1=0;
pr2=0;
for(t1=1;t1<1000;t1++){
if(mvnumber[t1]>0)pr1=pr1+1;
}

for(t1=1;t1<1000;t1++){
h=0;
if(mvnumber[t1]>0)h=h+1;
if(mv[t1][1]==m1)h=h+1;
if(h==2)pr2=pr2+1;
}

if(pr1>0)px=pr2/pr1;
if(px<0.01)px=0.01;
return px;
}

void readfile(String file){

String st;
BufferedReader br;

s=0;

try {
br = new BufferedReader(new InputStreamReader(new FileInputStream(file),"SJIS"));  
while((st = br.readLine()) != null) {  
s=s+1;
data[s]=st;  
}
br.close();
} catch (IOException e) {System.out.println(e);}  

datanumber=s;

}



}