#!/usr/local/bin/perl

##
## urlindex.pl  (C) 1995 Andrew Scherpbier
##
## This program will build an index of all the URLs in the
## htdig document database.
##

use GDBM_File;
require('SDSU_www.pl');

$dbfile = "/gopher/www/htdig/sdsu3d.docdb";
$dbfile = "/tmp/db.docdb";
$exclude = "rohan.sdsu.edu\\/home\\/";

tie(%docdb, GDBM_File, $dbfile, GDBM_READER, 0) || die "Unable to open $dbfile: $!";

print "Reading...\n";

##
## Read in all the relevant data.
##
while (($key, $value) = each %docdb)
{
    next if $key =~ /^nextDocID/;
    %record = parse_ref_record($value);
    next if $record{"STATE"} eq 1;
    next if $key =~ /$exclude/;

    $title = $record{"TITLE"};
    
    ##
    ## Get rid of starting and trailing whitespace junk
    ##
    $title =~ s/^[ \t\n\r]*//;
    $title =~ s/[ \t\n\r]*$//;
    
    ##
    ## If the title starts with 'the', it will be taken out and added
    ## to the end of the title.  This means that a title like "The
    ## Homepage of X" will become "Homepage of X, The"
    ##
    if ($title =~ /^the /i)
    {
	$title = substr($title, 4) . ", " . substr($title, 0, 3);
    }
    if ($title =~ /^SDSU /)
    {
	$title = substr($title, 5) . ", " . substr($title, 0, 4);
    }
    if ($title =~ /^San Diego State University /i)
    {
	$title = substr($title, 27) . ", " . substr($title, 0, 26);
    }
    $value = $title;
    $value =~ tr/A-Z/a-z/;
    $titles{$value} = "$title\001$key";
    push(@unsorted, $value);
}

$current = " ";
open(M, ">index.html");
print M "<html><head><title>Index of all documents at SDSU</title></head>\n";
print M "<body>\n";
print M &www_logo_2("Index of all documents at SDSU");
print M "<p>This is a list of WWW documents that were found while indexing all\n";
print M "the publicly available WWW servers at San Diego State University.\n";
print M "The documents are indexed by their titles.\n";
print M "</p><h2>\n";

$previous = "";

print "Writing...\n";

foreach $value (sort @unsorted)
{
    next if $value eq $previous;
    $previous = $value;
    next if !($value =~ /^[a-zA-Z]/);

    ($title, $url) = split('\001', $titles{$value}, 2);

    $first = substr($title, 0, 1);
    if ($current =~ /$first/i)
    {
	print F "<li><a href=\"$url\">$title</a></li>\n";
    }
    else
    {
	##
	## New letter.  Open a new file for it
	##
	$current = $first;
	$current =~ tr/a-z/A-Z/;
	print F "</li></body></html>\n";
	close(F);
	open(F, ">index$current.html");
	print F "<html><head><title>Index for $current</title></head>\n";
	print F "<body>\n";
	print F &www_logo_2("Index for $current");
	print F "<ul>\n";
	print F "<li><a href=\"$url\">$title</a></li>\n";

	##
	## Add a reference to the main index for this letter
	##
	print M " <a href=\"index$current.html\">$current</a>\n";

	print "Index of $current\n";
    }
}

close(F);

print M "</h2></body></html>\n";
close(M);


sub parse_ref_record
{
    local($value) = @_;
    local(%rec, $length, $count, $result);

    while (length($value) > 0)
    {
	$what = unpack("C", $value);
	$value = substr($value, 1);
	if ($what == 0)
	{
	    # ID
	    $rec{"ID"} = unpack("i", $value);
	    $value = substr($value, 4);
	}
	elsif ($what == 1)
	{
	    # TIME
	    $rec{"TIME"} = unpack("i", $value);
	    $value = substr($value, 4);
	}
	elsif ($what == 2)
	{
	    # ACCESSED
	    $rec{"ACCESSED"} = unpack("i", $value);
	    $value = substr($value, 4);
	}
	elsif ($what == 3)
	{
	    # STATE
	    $rec{"STATE"} = unpack("i", $value);
	    $value = substr($value, 4);
	}
	elsif ($what == 4)
	{
	    # SIZE
	    $rec{"SIZE"} = unpack("i", $value);
	    $value = substr($value, 4);
	}
	elsif ($what == 5)
	{
	    # LINKS
	    $rec{"LINKS"} = unpack("i", $value);
	    $value = substr($value, 4);
	}
	elsif ($what == 6)
	{
	    # IMAGESIZE
	    $rec{"IMAGESIZE"} = unpack("i", $value);
	    $value = substr($value, 4);
	}
	elsif ($what == 7)
	{
	    # HOPCOUNT
	    $rec{"HOPCOUNT"} = unpack("i", $value);
	    $value = substr($value, 4);
	}
	elsif ($what == 8)
	{
	    # URL
	    $length = unpack("i", $value);
	    $rec{"URL"} = unpack("x4 A$length", $value);
	    $value = substr($value, 4 + $length);
	}
	elsif ($what == 9)
	{
	    # HEAD
	    $length = unpack("i", $value);
	    $rec{"HEAD"} = unpack("x4 A$length", $value);
	    $value = substr($value, 4 + $length);
	}
	elsif ($what == 10)
	{
	    # TITLE
	    $length = unpack("i", $value);
	    $rec{"TITLE"} = unpack("x4 A$length", $value);
	    $value = substr($value, 4 + $length);
	}
	elsif ($what == 11)
	{
	    # DESCRIPTIONS
	    $count = unpack("i", $value);
	    $value = substr($value, 4);
	    $result = "";
	    foreach (1 .. $count)
	    {
		$length = unpack("i", $value);
		$result = $result . unpack("x4 A$length", $value) . "";
		$value = substr($value, 4 + $length);
	    }
	    chop $result;
	    $rec{"DESCRIPTIONS"} = $result;
	}
	elsif ($what == 12)
	{
	    # ANCHORS
	    $count = unpack("i", $value);
	    $value = substr($value, 4);
	    $result = "";
	    foreach (1 .. $count)
	    {
		$length = unpack("i", $value);
		$result = $result . unpack("x4 A$length", $value) . "";
		$value = substr($value, 4 + $length);
	    }
	    chop $result;
	    $rec{"ANCHORS"} = $result;
	}
	elsif ($what == 13)
	{
	    # EMAIL
	    $length = unpack("i", $value);
	    $rec{"EMAIL"} = unpack("x4 A$length", $value);
	    $value = substr($value, 4 + $length);
	}
	elsif ($what == 14)
	{
	    # NOTIFICATION
	    $length = unpack("i", $value);
	    $rec{"NOTIFICATION"} = unpack("x4 A$length", $value);
	    $value = substr($value, 4 + $length);
	}
	elsif ($what == 15)
	{
	    # SUBJECT
	    $length = unpack("i", $value);
	    $rec{"SUBJECT"} = unpack("x4 A$length", $value);
	    $value = substr($value, 4 + $length);
	}
	elsif ($what == 16)
	{
	    # STRING (ignore, but unpack)
	    $length = unpack("i", $value);
	    $rec{"STRING"} = unpack("x4 A$length", $value);
	    $value = substr($value, 4 + $length);
	}
	elsif ($what == 17)
	{
	    # METADSC
	    $length = unpack("i", $value);
	    $rec{"METADSC"} = unpack("x4 A$length", $value);
	    $value = substr($value, 4 + $length);
	}
	elsif ($what == 18)
	{
	    # BACKLINKS
	    $rec{"BACKLINKS"} = unpack("i", $value);
	    $value = substr($value, 4);
	}
	elsif ($what == 19)
	{
	    # SIGNATURE
	    $rec{"SIG"} = unpack("i", $value);
	    $value = substr($value, 4);
	}
    }
    print "title = $rec{'TITLE'}\n";
    return %rec;
}







