Attachment 596212 Details for Bug 837626 – simple parse of inverted index file (cscope.in.out)

simple parse of inverted index file (cscope.in.out)

index.pl (text/plain), 3.19 KB, created by Paul Bolle on 2012-07-04 12:50:26 UTC

(hide)

Description:

Filename:

MIME Type:

Creator: Paul Bolle

Created: 2012-07-04 12:50:26 UTC

Size: 3.19 KB

patch

obsolete

>use 5.014;
>use warnings;
>use autodie;
>
># XXX x86_64 only (unpack( "Q<", [...] ) will probably fail on x86 anyway)
>my $long = 8;
>
>exit main ( @ARGV );
>
>sub parse_entry_idx {
>	my ($index, $start) = @_;
>
>	my ($tmp, %elem);
>
>	read( $index, $tmp, 2 );
>	$elem{'offset'} = $start + unpack( "S<", $tmp );
>	read( $index, $tmp, 1 );
>	$elem{'size'} = unpack( "C", $tmp );
>	read( $index, $tmp, 1 );
>	$elem{'space'} = unpack( "C", $tmp );
>	# padding
>	read( $index, $tmp, 4 );
>	read( $index, $tmp, $long );
>	$elem{'post'} = unpack( "Q<", $tmp );
>
>	%elem;
>}
>
>sub get_entry {
>	my ($index, %elem) = @_;
>
>	my ($tmp, %entry);
>
>	$entry{'count'} = $elem{'post'};
>	seek( $index, $elem{'offset'}, 0 );
>	read( $index, $tmp, $elem{'size'} );
>	$entry{'value'} = unpack( "a*", $tmp );
>	# round up to next long boundary
>	my $padding = (int(($elem{'size'} + 7) / 8)) * 8 - $elem{'size'};
>	$entry{'padding'} = $padding;
>	read( $index, $tmp, $padding );
>	read( $index, $tmp, $long );
>	
>	# offset of first entry in postings file
>	$entry{'offset'} = unpack( "Q<", $tmp );
>
>	%entry;
>}
>
>sub parse_block {
>	my ($index, $current, $uniq, $total, %header) = @_;
>
>	my ($count, $next, $prev, @entry_idx, $end_of_idx);
>
>	my $start = $header{'cntlsize'} + $header{'sizeblk'} * $current;
>	seek( $index, $start, 0 );
>	read( $index, $count, $long );
>	$count = unpack( "Q<", $count );
>	$uniq += $count;
>	read( $index, $next, $long );
>	$next = unpack( "Q<", $next );
>	read( $index, $prev, $long );
>	$prev = unpack( "Q<", $prev );
>	# block starts with 3 longs: count, next, prev (24 bytes)
>	# size of elem: 2 + 1 + 1 + 4 + 8 (16 bytes) 
>	$end_of_idx = $start + 3 * $long + ($count * 2 * $long);
>	while ($count) {
>		my %elem = parse_entry_idx( $index, $start );
>		push( @entry_idx, \%elem );
>		$count -= 1;
>		if ($elem{'offset'} < $end_of_idx) {
>			my $offset = $elem{'offset'};
>			print STDERR "corruption in block $current, starting ";
>			printf STDERR "at offset 0x%04x", $offset;
>			printf STDERR " (end of index: 0x%04x)\n", $end_of_idx;
>		}
>	}
>
>	foreach my $elem (@entry_idx) {
>		if ($elem->{'size'}) {
>			my %entry;
>			(%entry) = get_entry( $index, %$elem );
>			$total += $entry{'count'};
>		}
>	}
>
>	($next, $uniq, $total);
>}
>
>sub parse_header {
>	my ($index) = @_;
>
>	my ($tmp, %header);
>	read( $index, $tmp, $long );
>	$header{'version'} = unpack( "Q<", $tmp );
>	die
>		unless ($header{'version'} == 1);
>	read( $index, $tmp, $long );
>	$header{'filestat'} = unpack( "Q<", $tmp );
>	read( $index, $tmp, $long );
>	$header{'sizeblk'} = unpack( "Q<", $tmp );
>	read( $index, $tmp, $long );
>	$header{'startbyte'} = unpack( "Q<", $tmp );
>	read( $index, $tmp, $long );
>	$header{'supsize'} = unpack( "Q<", $tmp );
>	read( $index, $tmp, $long );
>	$header{'cntlsize'} = unpack( "Q<", $tmp );
>	read( $index, $tmp, $long );
>	$header{'share'} = unpack( "Q<", $tmp );
>
>	%header;
>}
>
>sub main {
>	my ($path) = @_;
>
>	open( my $index, '<', $path );
>	my %header = parse_header( $index );
>
>	my $next;
>	my $current = 0;
>	my $uniq = 0;
>	my $total = 0;
>	while (1) {
>		($next, $uniq, $total) = parse_block( $index, $current, $uniq, $total, %header );
>		last
>			unless ($next > $current);
>		$current = $next;
>	}
>	close( $index );
>
>	# drop the sole placeholder term, to match db header
>	$uniq -= 1;
>	say "terms: $uniq";
>	say "posts: $total";
>
>	0;
>}

Actions: View

Attachments on bug 837626: 596212 | 596217 | 596219