diverse speedups, reduction of file io, ...

1 files changed, 77 insertions, 49 deletions

diff --git a/get-nodes.pl b/get-nodes.pl
index 7ee1923..af89615 100755
--- a/get-nodes.pl
+++ b/get-nodes.pl
@@ -1,10 +1,19 @@
 #!/usr/bin/perl -wT
+use strict;
+use warnings;
+use utf8;
+use JSON;
 use MediaWiki::API;
+use Text::Unidecode;
 
-$ENV{HOME} =~ m!^(/.+)$!;
+binmode STDERR, ':utf8';
+
+$ENV{HOME} =~ m!\A(/.+)\Z!;
 my $HOME = $1;
 
-my $DEBUG = (defined($ENV{DEBUG}) && $ENV{DEBUG} =~ m!^(\d+)$!) ?  $1 : 0;
+my $DEBUG = (defined($ENV{DEBUG}) && $ENV{DEBUG} =~ m!\A(\d+)\Z!) ?  $1 : 0;
+
+my $nodes;
 
 sub debug {
 	return unless $_[0] <= $DEBUG;
@@ -14,75 +23,94 @@ sub debug {
 
 my $mw = MediaWiki::API->new( { api_url => 'http://freifunk.in-kiel.de/mediawiki/api.php' } );
 
-# $mw->login( { lgname => 'WikiSysop', lgpassword => 'WikiPassFreifunk' } ) || die $mw->{error}->{code} . ': ' . $mw->{error}->{details};
-
-my $nodes = $mw->list ( {
-	action => 'query',
-	list => 'embeddedin',
-	einamespace => 0,
-	eilimit => 1024,
-	eifilterredir => 'nonredirects',
-	eititle => 'Template:Node'
-} ) || die $mw->{error}->{code} . ': ' . $mw->{error}->{details};
-
-foreach (@{$nodes}) {
-	debug(2, "title:", $_->{title});
-	my $tmp = $mw->api ( {
-		action => 'parse',
-		page => $_->{title},
-		prop => 'wikitext',
-	} ) || die $mw->{error}->{code} . ': ' . $mw->{error}->{details};
-	$tmp = $tmp->{parse}->{wikitext}->{'*'};
-	debug(5, "tmp:", $tmp);
+sub handle_node {
+	my($node) = @_;
+	my($title,$name,$gps,$mac,$vpn,$fastd,$fn,$f);
 
-	$tmp =~ m!^{{Node\s*(\|.+?)}}!sm;
-	$tmp = $1;
+	debug(2, "title:", $node->{title});
 
-	next unless defined($tmp);
+	$node->{title} =~ m!\ANodes/([ \w-]+)\Z! && ($title = $1);
 
-	%fields = (Node => split /^\|(\w+)\s*=\s*/m, $tmp, -1);
+	unless (defined($title)) {
+		debug(1, "no valid title:", $node->{title});
+		next;
+	}
 
-	foreach my $k (qw/NAME GPS MAC VPN FASTD/) {
-		next unless defined $fields{$k};
-		$fields{$k} =~ s/^\s+//mg;
-		$fields{$k} =~ s/\s+$//mg;
-		debug(3, ">>>",$k, " = ", $fields{$k}, "<<<");
+	($name = lc(unidecode($title))) =~ s/\s/_/g;
+
+	unless ($name =~ /\A[0-9a-z_-]+\Z/) {
+		debug(1, "invalid name:", $name);
+		next;
 	}
 
-	my($name,$gps,$mac,$vpn,$fastd,$fn,$f);
+	my $tmp = $mw->get_page({ title => $node->{title} }) || die $mw->{error}->{code} . ': ' . $mw->{error}->{details};
+	$tmp = $tmp->{'*'};
+	debug(4, "tmp:", $tmp);
 
-	$name = $fields{NAME} if (defined($fields{NAME}) && $fields{NAME} gt "" && $fields{NAME} =~ /^[0-9A-Za-z_-]+$/);
-	$gps = $fields{GPS} if (defined($fields{GPS}) && $fields{GPS} gt "" && $fields{GPS} =~ /^\d{1,3}\.\d+\s+\d{1,2}\.\d+$/);
-	$mac = $fields{MAC} if (defined($fields{MAC}) && $fields{MAC} gt "" && $fields{MAC} =~ /^([0-9A-F]{2}:){5}([0-9A-F]{2})$/i);
-	$vpn = $fields{VPN} if (defined($fields{VPN}) && $fields{VPN} gt "");
-	$fastd = $fields{FASTD} if (defined($fields{FASTD}) && $fields{FASTD} gt "" && $fields{FASTD} =~ /^key "[0-9a-f]{64}";$/m);
+	next unless $tmp =~ m!{{Node\s*(\|.+?)}}!sm;
+	$tmp = $1;
 
-	unless (defined($name)) {
-		debug(1, "no valid name:", $_->{title});
-		next;
+	debug(5, "tmp:", $tmp);
+
+	my %fields = (Node => split /^\|(\w+)\s*=\s*/m, $tmp, -1);
+
+	foreach my $k (qw/GPS MAC VPN FASTD/) {
+		next unless defined $fields{$k};
+		$fields{$k} =~ s/\A\s+//mg;
+		$fields{$k} =~ s/\s+\Z//mg;
+		debug(3, ">>>",$k, " = ", $fields{$k}, "<<<");
 	}
 
+	(defined($fields{GPS})) && ($fields{GPS} gt "") && ($fields{GPS} =~ m!\A(\d{1,3}\.\d+)[\., /]+(\d{1,2}\.\d+)\Z!) && ($gps = "$1 $2");
+	(defined($fields{MAC})) && ($fields{MAC} gt "") && ($fields{MAC} =~ /\A(([0-9A-Fa-f]{2}:?){5}([0-9A-Fa-f]{2}))\Z/) && ($mac = lc($1)); # && ($mac =~ s/\A([0-9a-f]{2}:?){5}([0-9a-f]{2})\Z/$1:$2:$3:$4:$5:$6/e);
+	(defined($fields{VPN})) && ($fields{VPN} gt "") && ($vpn = $fields{VPN});
+	(defined($fields{FASTD})) && ($fields{FASTD} gt "") && ($fields{FASTD} =~ /\Akey\s*"[0-9a-f]{64}";\Z/m) && ($fastd = $fields{FASTD}) && ($fastd =~ s/key\s*"/key "/g);
+
 	if (defined($fastd)) {
 		$fn = "/etc/fastd/ffki/peers/$name";
 		unless (-f $fn) {
 			open($f,">",$fn);
+			binmode $f, ':utf8';
 			print $f $fastd."\n";
 			close($f);
 		}
 	}
 
 	if (defined ($mac)) {
-		$mac =~ tr/A-Z/a-z/;
-		open($f,">","/opt/ffmap/aliases.json.d/$mac");
-		print $f '   "'.$mac.'" : {'."\n";
-		print $f '      "name" : "'.$name.'"'.",\n" if defined($name); 
-		print $f '      "gps" : "'.$gps.'"'.",\n" if defined($gps); 
-		print $f '      "vpn" : "true"'.",\n" if defined($vpn); 
-		print $f '      "": ""'."\n"; 
-		print $f '   },'."\n";
-		close($f);
+		my $node;
+		$node->{'name'} = $title if defined $title;
+		$node->{'gps'} = $gps if defined $gps;
+		$node->{'vpn'} = 'true' if defined $vpn;
+		$nodes->{$mac} = $node;
+	}
+}
+
+sub handle_nodes {
+	my ($ref) = @_;
+	foreach my $node (@$ref) {
+		&handle_node($node);
 	}
 }
 
+$mw->list (
+	{
+		action => 'query',
+		list => 'embeddedin',
+		einamespace => 0,
+		eilimit => 'max',
+		eifilterredir => 'nonredirects',
+		eititle => 'Template:Node'
+	}, {
+		max => 'max',
+		hook => \&handle_nodes
+	}
+) || die $mw->{error}->{code} . ': ' . $mw->{error}->{details};
+
+open(my $f,">","/opt/ffmap/aliases.json.new");
+print $f encode_json($nodes);
+close($f);
+
+rename "/opt/ffmap/aliases.json.new", "/opt/ffmap/aliases.json";
+
 __END__