autodoc/bin/install_dicts.pl

#!/usr/bin/perl

use strict;
use lib '/opt/autodoc/lib';
use Autodoc;
use utf8;
use warnings;

$|=1;

my $conf = load_conf("../etc/autodoc.json");
my $dbh  = sqlconnect($conf->{sql});
my $chunk = 10000;

my %lang = (
	'fr' => [ 'french' ],
	'de' => [ 'swiss', 'ngerman' ],
	'en' => [ 'british-english-large', 'american-english-large' ],
	'it' => [ 'italian' ],
);

foreach my $lang ( sort keys %{$conf->{dict}} ) {
	print "Loading language $lang ...\n";
	foreach my $dict ( @{$conf->{dict}{$lang}} ) {
		my $file = '/usr/share/dict/'.$dict;

		my @words;
		open(DICT, $file) || die "Failed to load $file, did you install the required dictionaries?";
		while(<DICT>) { chomp; push @words, $_; }
		close(DICT);

		my $len = scalar(@words);

		for(my $pos = 0; $pos < $len; $pos+=$chunk) {
			my @query;
			my @args;

			for(my $i=0; $i<$chunk && defined $words[$pos+$i]; $i++) {
				push @args, $words[$pos+$i];
				push @args, $lang;
				push @query, "(?,?)";
			}

			my $query = "INSERT IGNORE INTO dict" .
				"(word, lang) VALUES " .
				join(",",@query);

			sqlquery($dbh, $query, @args);

			my $cnt = $pos + (scalar(@args)/2);

			printf("\t%s %s/%s (%i%%)    \r",
				$dict, $cnt , $len, int($cnt/$len*100) );
		}
		printf("\n");
	}
}