autodoc/bin/load_dicts.pl

#!/usr/bin/perl

use strict;
use JSON;
use DBI;
use utf8;
use GD::Simple;
use Data::Dumper;
use Data::UUID;
use warnings;

$|=1;

$Data::Dumper::Sortkeys = 1;

my $conf = load_conf("../etc/autodoc.json");
my $dbh  = sqlconnect($conf->{sql});

my %lang = (
	'fr' => [ 'french' ],
	'de' => [ 'swiss', 'ngerman' ],
	'en' => [ 'british-english-large', 'american-english-large' ],
	'it' => [ 'italian' ],
);

foreach my $lang ( sort keys %lang ) {
	print "Loading language $lang ...\n";
	foreach my $dict ( @{$lang{$lang}} ) {
		my $file = '/usr/share/dict/'.$dict;
		my $len;

		open(LEN,"wc -l $file |");
		while(<LEN>) { if ( /(\d+)/ ) { $len = $1; } }
		close(LEN);

		next if !defined $len;
		my $cnt = 0;

		my $start = time();
		open(DICT,$file);
		while(<DICT>) {
			chomp;
			sqlquery($dbh, "INSERT IGNORE INTO dict SET word = ?, lang = ?",
				$_, $lang);
			$cnt++;
			if ( ! ( $cnt % 777 ) || $cnt eq $len ) {
				printf("\t%s %s/%s (%i%%) ETA: %ss    \r",
					$dict, $cnt, $len, int($cnt/$len*100),
					time()-$start eq 0 ? '-' : int( ( (time()-$start)/$cnt*$len ) - ( time()-$start ) )
				);
			}
		}
		close(DICT);
		printf("\n");
	}
}

sub load_conf {
	my($file) = @_;

	my $x='';

	open(F,"$file") || die "Failed to load configuration file";
	while(<F>) { $x.=$_; }
	close(F);

	return from_json($x);
}

sub sqlconnect {
	my($sql) = @_;

	my $dsn = "DBI:mysql:database=$sql->{base};host=$sql->{host}";
	my $dbh = DBI->connect($dsn, $sql->{user}, $sql->{pass}, {
		mysql_enable_utf8 => 1
	}) || die "Failed to connect to database";

	return $dbh;
}

sub sqlquery {
	my $dbh   = shift;
	my $query = shift;
	my @args  = @_;

	#print STDERR "$query\n";

	my $sth = $dbh->prepare($query) || die "Failed to execute SQL query";
	$sth->execute(@args) || die "Failed to execute SQL query";
	return $sth;
}