#!/usr/bin/perl


=head1 NAME

digestunstable.pl - Compiles the Nettime Unstable Digest

=head1 SYNOPSIS 

digestunstable.pl B<directory>

=head1 DESCRIPTION

Perl script for compiling I<nettime unstable digest>,
the unstable digest on the international net culture
mailing list Nettime-l.

The script processes a directory containing the
single text files for a digest, includes them
according to the alphabetical order of the filenames,
strips the input from redundant headers, MIME
codes and HTML attachments and appends a 
subject/sender index.

=head1 EXAMPLES

B<digestunstable.pl I<material_6>> 

Open all files in the directory B<material_6> and
print a nettime unstable digest vol. 6 in plain text format 
to the standard output.

B<digestunstable.pl -f html I<material_6a>> 

Open all files in the directory B<material_6a> and
print an artcom unstable digest vol. 6 in html format
as separate files in a directory artcom-unstable-digest_6


=head1 LICENSE

GNU General Public License, see accompanying file COPYING

=head1 AUTHOR

Florian Cramer <cantsin@zedat.fu-berlin.de>

=head1 RELEASE

$Id: digestunstable.pl,v 1.13 2003/01/26 18:51:21 paragram Exp $

=cut

############################################
#
#  Setup
#
############################################




# Parse command line arguments && determine output format

$format = 'txt';
use Getopt::Std;
getopts('f:h');
if ($opt_f) {
	if ($opt_f =~ /htm/i) {$format = 'html'}
	}
if ($opt_h == 1) {&abort}

# name of the target forum
$target = "nettime";

# name of this file
$name_of_script = 'digestunstable.pl';

# directory containing configuration files
$config_dir = './config';

# name of the toc database file
$database_file = 'index.rdb';

# configuration file listing the digest editors
$editors_db = 'editors.txt';

# configuration file listing mailing list-specific "Subject:" line prefixes 
$list_prefixes_db = 'list-prefixes.txt';

# configuration file listing mailing list-specific footer lines
$list_footers_db = 'list-footers.txt';

# configuration file listing mailing list-specific "Sender:"lines
$list_senders_db = 'list-senders.txt';

# Get the name of the directory containing the texts to include
$source_dir = $ARGV[0];
if (not $source_dir) {
	&abort;
	}
# if the directory name has an "a" suffix, target "artcom" instead of
# "nettime"
elsif ($source_dir =~ /a[\/]*$/) {
	$target = 'artcom';
	}
# Get the digest volume number from the name of the directory
# (like: "material_5" ==> vol. 5)
$digest_volume = $source_dir;
$digest_volume =~ s/[^0-9]//g;
$digest_volume =~ s/^[0]+//;

# Create target directory name from digest volume (applies only for HTML
# versions of the digest

if ($format eq 'html') {
	$target_dir = $target . '-unstable-digest_' . $digest_volume;
	mkdir ($target_dir) || die ("Couldn't create directory $target_dir\n"); 
	print "Created directory $target_dir\n";
	# Read table of contents data base
	open (DATABASE, "<$database_file");
	while (<DATABASE>) {
		unless (/unstable-digest_$digest_volume\/[0-9]+\.html/) {
			push @database, $_;
			}
		}
	close (DATABASE);
	if ($#database <= 1) {
		push @database, "VOLUME\tARTNO\tAUTHOR\tADDRESS\tSUBJECT\tFILE\n";
		push @database, "------\t-----\t------\t-------\t-------\t----\n";
		}
	}

# Get today's date
$date = localtime;

# Editors

open (EDITORS, "<$config_dir/$editors_db") || die ("configuration file $config_dir/$editors_db not found.\n");
while (<EDITORS>) {
	# ignore comments
	unless (/^#/ || /^[\s]*$/) {
		chomp;
		@input = split (':');
		if ($input[0] eq 'NAME') {$line = $input[1]}
		elsif ($input[0] eq 'FIRSTNAME') {$line .= ",$input[1]"}
		elsif ($input[0] eq 'EMAIL') {$line .= ",$input[1]"}
		elsif ($input[0] eq 'LISTS') {
			$input[1] =~ s/,/:/g;
			$line .= ",$input[1]";
			push @editors, $line;
			}
		}
	}
close (EDITORS);
@editors = sort(@editors);

# Mailing list prefixes
# (regular expressions!)
open (LIST_PREFIXES, "<$config_dir/$list_prefixes_db") || die ("configuration file $config_dir/$list_prefixes_db not found.\n");
while (<LIST_PREFIXES>) {
	chomp;
	/^([^:]+):(.*)/;
	push @list_prefixes, $2;
	# push @list_prefixes_id, $1;
	}
close (LIST_PREFIXES);

# Mailing list footers 
# (regular expressions!)
open (LIST_FOOTERS, "<$config_dir/$list_footers_db") || die ("configuration file $config_dir/$list_footers_db not found.\n");
while (<LIST_FOOTERS>) {
	chomp;
	/^([^:]+):(.*)/;
	$list_footers{$1} = $2; 
	}
close (LIST_FOOTERS);

# Mailing list Sender: ids
# (regular expressions!)
open (LIST_SENDERS, "<$config_dir/$list_senders_db") || die ("configuration file $config_dir/$list_senders_db not found.\n");
while (<LIST_SENDERS>) {
	chomp;
	/^([^:]+):(.*)/;
	push @list_senders, $2;
	push @list_senders_id, $1;
	}
close (LIST_SENDERS);


# The following variable determines whether an input line should be kept or not
# Possible values:
#
# -1  : Suppress the current line and all following lines (until the flag
#       is set to a different value)
#  0  : Suppress the current line only
#  1  : Print the current line
$print_flag = -1;



############################################
#
#  Open and filter the mail to be included
#
############################################

# All files in the source directory will be included
# The order of inclusion follows the file names(!!!)
opendir (SOURCEDIR, "$source_dir") || die ("Directory $source_dir does not exist.\n");
@files = sort (readdir (SOURCEDIR));
closedir (SOURCEDIR);
foreach $file (@files) {
	# Exclude dotfiles like "." and ".."
	if ($file =~ /\w/) {
		if ($format eq "html") {
			$article_number++;
			$html_file = $article_number;
			if ($article_number < 10) {$html_file = "0$html_file"}
			$html_file = "$html_file.html";
			push @html_files, $html_file;
			open (HTML_FILE, ">$target_dir/$html_file") || die ("Couldn't open $target_dir/$html_file");
			print HTML_FILE "<html>\n";
			print HTML_FILE "<pre>";
			}
		# we don't know yet which mailing list this message
		# came from
		$listname = ''; 
		$total_message_counter++;
		$current_message_part = 'header';
		open (TEXTFILE, "<$source_dir/$file");
		while (<TEXTFILE>) {
			chomp;
			if ($current_message_part eq 'header') {
				# Suppress all header lines except "From:",
				# "To:", "Date:" and "Subject:" and filter out 
				# non-Nettime subject line prefixes 
				$print_flag = 0;
				if (/^From:/) {
					# We want to print this out
					$print_flag = 1;
					# Clean up irregular From: lines
					s/^From:[\s]+/From: /;
					# Memorize this line for the appended
					# index
					push @from_line, $_;
					}
				elsif (/^To:/) {
					# We want to print this out
					$print_flag = 1;
					# Clean up irregular To: lines
					s/^To:[\s]+/To: /;
					# Memorize this line for the appended
					# index
					push @to_line, $_;
					}
				elsif (/^Subject:/) {
					# We want to print this out
					$print_flag = 1;
					# Clear list-specific prefixes
					foreach $list_prefix(@list_prefixes) {
						if (/$list_prefix/) {
							s/$list_prefix//;
							}
						}
					# Clean up irregular From: lines
					s/^Subject:[\s]+/Subject: /;
					# Memorize this line for the appended
					# index
					push @subject_line, $_;
					}
				elsif (/^Date:/) {$print_flag = 1}
				# Print "To:" line only if the message was sent
				# to Nettime
				elsif (/^To:/ && /nettime-l/) {$print_flag = 1}
				# Scan "Sender:" to find out which
				# mailing list the Mail is from
				elsif (/^Sender: /) {
					for ($x=0; $x<=$#list_senders; $x++) {
						if (/$list_senders[$x]/) {$listname = $list_senders_id[$x]}
						}
					}
				# If the line is empty, we're at the end of the
				# message header
				elsif (/^[\s]*$/) {
					$current_message_part = 'body'; 
					}
				}
			if ($current_message_part eq 'body') {
				# As opposed to the header, the default mode is
				# to not suppress the input -
				unless ($print_flag == -1) {$print_flag = 1}
				# Try to filter out MIME codes and HTML
				# attachments as good as possible
				if (/--=====================_.*ALT/ || /Content-Type: text\/plain/) {$print_flag = 0}
				elsif (/^This is a multi-part message in MIME format./) {$print_flag = 0}
				elsif (/^------=_NextPart_/) {$print_flag = 0}
				elsif (/^[\s]*charset=\"iso-/) {$print_flag = 0}
				elsif (/^Content-Transfer-Encoding:\ /) {$print_flag = 0}
				elsif (/^Content-Type: text\/html/) {$print_flag = -1}
				elsif (/^<!DOCTYPE HTML PUBLIC/) {$print_flag = -1}
				elsif (/^_______________________________________________$/) {$print_flag = 0}
				elsif ($print_flag == -1 && (/^<\/html>/ || /<\/HTML>$/)) {$print_flag = 0}
				# Scan for first footer line of the identified list
				# and filter out all successive lines
				if ($list_footers{$listname} && /$list_footers{$listname}/) {
					$print_flag = -1;
					}
				
				}
			if ($print_flag	== 1) {
				if ($format eq "txt") {print $_,"\n"}
				elsif ($format eq "html") {
					print HTML_FILE &txt2entities($_),"\n";
					}
				}
			}
		close (TEXTFILE);
		if ($format eq "html") {
			# print HTML footer
			print HTML_FILE "</pre>\n";
			print HTML_FILE "<hr>\n";
			print HTML_FILE "Back to <a href=\"index.html#$article_number\">$target unstable digest vol $digest_volume</a>\n";
			print HTML_FILE "</html>\n";
			close (HTML_FILE);
			# append author & subject info to database
			$author = $from_line[$#from_line];
			$author =~ s/^From: //;
			$author =~ s/(\<[^>]+\>)$//;
			$address = $1;
			$author =~ s/"//g;
			$address =~ s/[<>]//g;
			$subject = $subject_line[$#subject_line];
			$subject =~ s/^Subject: //;
			$database_record = &detab($target)."\t".&detab($digest_volume)."\t".&detab($article_number)."\t".&detab($author)."\t".&detab($address)."\t".&detab($subject)."\t".&detab("$target_dir/$html_file")."\n";
			push @database, $database_record;
			} 
		else {
			print "\n\n";
			}
		}
	}

############################################
#
#  Print digest index / footer
#
############################################

if ($format eq 'txt') {
	print "\n\n";
	print "$target unstable digest vol $digest_volume\n";
	print $date,"\n\n\n";

	for ($x=0; $x<$total_message_counter; $x++) {
		print $subject_line[$x], "\n";
		print '    ',$from_line[$x],"\n";
		print '    ',$to_line[$x],"\n\n";
		}
	print "\n\nlurking editors\n\n";

	foreach $editor(@editors) {
		@editor_data = split (',', $editor);
		@lists = split (':', $editor_data[$#editor_data]);
		@lists = sort (@lists);
		print "\l$editor_data[1] \l$editor_data[0] <$editor_data[2]>\n    ";
		foreach $list(@lists) {
			print $list, ' ';
			}
		print "\n";
		}
	print '$Id: digestunstable.pl,v 1.13 2003/01/26 18:51:21 paragram Exp $',"\n";
	}

elsif ($format eq "html") {
	open (HTML_FILE, ">$target_dir/index.html");
	print HTML_FILE "<html>\n";
	print HTML_FILE "<head>\n";
	print HTML_FILE "<title>$target unstable digest vol $digest_volume</title>\n";
	print HTML_FILE "</head>\n";
	print HTML_FILE "<body>\n\n";
	print HTML_FILE "<h1>$target unstable digest vol $digest_volume</h1>\n\n";
	print HTML_FILE "$date<p>\n\n";
	print HTML_FILE "<ul>\n\n";
	for ($x=0; $x<$total_message_counter; $x++) {
		print HTML_FILE '<li><a href="',$html_files[$x],'" name="', ($x+1), '">',&txt2entities($subject_line[$x]), "</a><br>\n";
		print HTML_FILE &txt2entities($from_line[$x]),"\n\n";
		}
	print HTML_FILE "</ul>\n\n";
	print HTML_FILE "<h2>lurking editors</h2>\n\n";
	print HTML_FILE "<ul>\n";
	foreach $editor(@editors) {
		@editor_data = split (',', $editor);
		@lists = split (':', $editor_data[$#editor_data]);
		@lists = sort (@lists);
		print HTML_FILE "<li>\l$editor_data[1] \l$editor_data[0] \&lt\;<a href=\"mailto:$editor_data[2]\">$editor_data[2]</a>\&gt\;: ";
		foreach $list(@lists) {
			print HTML_FILE $list, ' ';
			}
		print HTML_FILE "\n";
		}
	print HTML_FILE "</ul>\n";
	print HTML_FILE "<p>\n";
	print HTML_FILE '$Id: digestunstable.pl,v 1.13 2003/01/26 18:51:21 paragram Exp $',"\n";
	print HTML_FILE "<hr>\n";
	print HTML_FILE "Back to <a href=\"../index.html\">index of unstable digests</a><p>\n\n";
	print HTML_FILE "</body>\n";
	print HTML_FILE "</html>\n";
	close (HTML_FILE);
	print "$article_number files + 1 index file generated in $target_dir.\n";

	## write back database
	system ("mv $database_file $database_file.bak");
	# @database = sort(@database);
	open (DATABASE, ">$database_file");
	foreach $database_record(@database) {
		print DATABASE $database_record;
		}
	close (DATABASE);
	}

sub txt2entities {
		my $x = $_[0];
		$x =~ s/</\&lt\;/g;
		$x =~ s/>/\&gt\;/g;
		return $x;
	}

sub abort {
	die ("Usage: $name_of_script [-f txt|html] <directory>\n[directory name _must_ contain the digest's volume number!]\n"); 
}

sub detab {
	my $input = $_[0];
	$input =~ s/\t/ /g;
	return $input;
	}

# EOF



