User:Polbot/code/Function 2

For importing works and Wikifying them, the code has to be modified for each work. For each section below, the code given is a rather generic example, but you'll have to modify the code through trial and error to produce code that works for your source.

Gutenberg texts

# Note: This script is for works from Project Gutenberg, www.gutenberg.org, although it works pretty well for other texts.
#  I download the html or txt file, and place {{curly braces}} around chapter names to tell the script where to divide pages.
#  I also start each text file with a line containing the name of the work, and a line containing the name of the author.

use strict;
use LWP::UserAgent;
use Encode;
use Perlwikipedia;

my $file = shift;
my $test = shift;
my $pw;

print "Reading file. . .\n";
open(infile, "<$file") or die "File not found\n";

my $line = <infile>;
chop $line;
my $title = $line;

$line = <infile>;
chop $line;
my $author = $line;
my $translator = "";

if ($author =~ /^(.*)\|(.*)$/) {
	$author = $1;
	$translator = $2;
}

my @pagenames = ();
my @wikis = ();

$line = <infile>;
while ($line =~ /^\s*$/) {
	$line = <infile>;
}
chop $line;
	
$line =~ /^\{\{(.*)\}\}$/ or die "Improper format\n";

my $pagename = $1;
my $wiki = "";

while ($line = <infile>) {
	chop $line;
	
	if ($line =~ /^\{\{(.*)\}\}/) {
		my $temp = $1;
		
		push @pagenames, $pagename;
		push @wikis, $wiki;
		$pagename = $temp;
		$wiki = "";
	} else {
		$wiki .= "$line\n";		
	}
}

push @pagenames, $pagename;
push @wikis, $wiki;

close(infile);

print scalar(@pagenames) . " pages.\n";

if (!$test) {
	# login
	$pw=Perlwikipedia->new();
	#$pw->{debug} = 1;
	$pw->{mech}->agent('Firefox/2.0.0.6');
	$pw->set_wiki('en.wikisource.org', 'w');
	print "Logging in as Polbot\n";
	my $login_status=$pw->login('Polbot','[password]');
	die "I can't log in." unless $login_status == 0;
	
	# Output automatic TOC
	open(outfile, ">test.txt");
	print outfile "Contents:\n\n";
	foreach my $pagename (@pagenames) {
		if ($pagename =~ /^\#+/) {
			# if a chapter starts with #, it's its own page, not a subpage.
			# if it starts with ##, it's its own page, and it gets <poem> tags.
			if ($pagename =~ /^\#+(.*)\: (.*)$/) {
				print outfile "* [[$1]]: $2\n";
			} else {
				$pagename =~ /^\#+(.*)$/;
				print outfile "* [[$1]]\n";
			}
		} else {
			if ($pagename =~ /^(.*)\/(.*)$/) {
				my $book = $1;
				my $chapter = $2;

				if ($chapter =~ /^(.*)\: (.*)$/) {
					print outfile "* [[$title/$book/$1|$1]]: $2\n";
				} else {
					print outfile "* [[$title/$book/$chapter|$chapter]]\n";
				}
			} else {
				if ($pagename =~ /^(.*)\: (.*)$/) {
					print outfile "* [[$title/$1|$1]]: $2\n";
				} else {
					print outfile "* [[$title/$pagename|$pagename]]\n";
				}
			}
		}
	}
	close (outfile);		
	print "Wrote TOC\n";
	sleep 5;
}

foreach my $start (0 .. $#pagenames) {
	my $pagename = $pagenames[$start];
	my $wiki = $wikis[$start];
	my $poem = 0;
	$poem = 1 if $pagename =~ s/^\#\#/\#/;
	print " $start. " . $pagenames[$start] . "\n";
	
	$wiki =~ s/<\!--.*?-->//gi; # remove html comments
	$wiki =~ s/<span class=[\"\']?pagenum[\"\']?>.*?<\/span>//gi; # remove page numbers
	$wiki =~ s/<span class=[\"\']?smallcaps[\"\']?>(.*?)<\/span>/{{small-caps|$1}}/gi; # format smallcaps
	$wiki =~ s/<p class=[\"\']?authorsc[\"\']?>(.*?)<\/p>/<div align=right>{{bbsc|$1}}<\/div>/gi; # author smallcaps
	#$wiki =~ s/\s*<b>(.*?)<\/b>\s*/\n==$1==\n/gis; # Make bolds into headers
	$wiki =~ s/\s*<h\d[^>]*>\s*([^\s].*?[^\s])\s*<\/h\d>\s*/\n'''$1'''\n\n/gis; # Make headers into bold
	$wiki =~ s/<\/?u>//gi; # remove underlines
	$wiki =~ s/<ins.*?>(.*?)<\/ins>/$1/gis; #remove ins
	$wiki =~ s/<br( \/)?>//gi; # take out BR
	$wiki =~ s/<\/?(i|em)>/\'\'/gi; # italics
	$wiki =~ s/<hr[^>]*>//gi; # remove HRs. 
	$wiki =~ s/\&nbsp\;/ /g;
	$wiki =~ s/<div class=[\"\']?blockquot[\"\']?>(.*?)<\/div>/<div style=\"margin-left: 5%; margin-right: 10%;\">\n$1\n<\/div>/gis; # blockquotes
  #$wiki =~ s/<(\/?)pre>/<$1poem>/gi; # pre to poem
  
	# footnotes:
#	while ($wiki =~ /<a href=\"\#(.*?)\".*?<\/a>/gi) {
#		my $footnote_name = $1;
#
#		$wiki =~ s/<a href=\"\#$footnote_name\".*?<\/a>(.*)<div class=\"footnote\"><a name=\"$footnote_name\".*?<\/a>.*?<\/a>\s*(.*?)<\/div>/<ref>$2<\/ref>$1/si;
#	}
#	$wiki =~ s/<hr( style=\"[^\"]*\" )?\/>\n\'\'\'FOOTNOTES:\'\'\'//gis;

	# footnotes: 
	#    <a name="Ibr" id="Ibr"></a><a href="#Ib"><sup>b</sup></a>
	#    <p class="footnote"><a name="Ia" id="Ia"></a>\n<a href="#Iar">a</a>&nbsp;&nbsp;&nbsp;See <a href="#pagex">page x</a>.</p>
	while ($wiki =~ /<a href=\"\#(.*?)\".*?<\/a>/gi) {
		my $footnote_name = $1;

		$wiki =~ s/<a href=\"\#$footnote_name\".*?<\/a>(.*)<p class=\"footnote\"><a name=\"$footnote_name\".*?<\/a>.*?<\/a>\s*(.*?)<\/p>/<ref>$2<\/ref>$1/si;
	}
		
	# <span class="i0">And<a name="FNanchor_46:2_77" id="FNanchor_46:2_77"></a><a href="#Footnote_46:2_77" class="fnanchor">[46:2]</a></span><br />

	# poems:
	#$wiki =~ s/<p class=\"i0">(.*?)<\/p>/    $1/gis;
	$wiki =~ s/^<span class="i1">(.*)<\/span>/   $1/gim;
	#$wiki =~ s/<p class=\"i8">(.*?)<\/p>/            $1/gis;
	$wiki =~ s/<div class=\"stanza\">(.*?)<\/div>/$1/gis;
	$wiki =~ s/<div class=\"poem\">(.*?)<\/div>/<poem>$1<\/poem>/gis;
	
	# other divs and spans
	$wiki =~ s/<\/?p*?>/\n/gi; # remove paragraphs
	$wiki =~ s/^(\s*\*)+\s*$/{{star-divider}}/gm; #  * * *
	$wiki =~ s/\n?<a [^>]*?\/>/ /gi; # remove lone anchors
	$wiki =~ s/<a .*?>(.*?)<\/a>/$1/gi; # links
	$wiki =~ s/<img .*?>/\n[image not included]\n/gi; # images
	$wiki =~ s/\_/''/g; # convert _ into italics
	#$wiki =~ s/^ +//gm if !$poem; #leading spaces 
	
	$wiki =~ s/\n\s*\n\s*\n/\n\n/g; # Take out excess whitespace	
	$wiki = "<poem>\n$wiki</poem>\n[[Category:Poems]]\n" if $poem;
	
	#	$wiki .= "\n==Footnotes==\n" . '<div style="-moz-column-count: 2; column-count: 2;">' . "<references /></div>" if $wiki =~ /<ref>/;
	$wiki .= "\n==Footnotes==\n<references />" if $wiki =~ /<ref>/;
	
	my $header = "{{header2\n|title=[[$title]]\n|author=$author\n";
	$header .= "|translator=$translator\n" if $translator;
	
	my $prev = "";
	if ($start > 0) {
		$prev = $pagenames[$start-1] ;
		$prev =~ s/\: .*$//;
	}
	my $next = $pagenames[$start+1];
	$next  =~ s/\: .*$//;
	
	# if a chapter starts with #, it's its own page, not a subpage.
	if ($prev) {
		if ($prev =~ /^\#+(.*)$/) {
			$prev = "[[$1]]";
		} else {
			$prev = "[[$title/$prev|$prev]]";
		}
	}

	if ($next) {
		if ($next =~ /^\#+(.*)$/) {
			$next = "[[$1]]";
		} else {
			$next = "[[$title/$next|$next]]";
		} 
	}
	$pagename =~ /\#*(.*)/;
	$header .= "|section=$1\n";
	$header .= "|previous=$prev\n|next=$next\n|notes=\n}}\n";

	$wiki = $header . $wiki;
	#$wiki = "$header<poem>\n$wiki</poem>";  # THIS TIME ONLY
	
	if ($test) {
		open(outfile, ">test.txt");
		print outfile $wiki;
		die "done";
	} else {
		my $writeto = $pagename;
		$writeto =~ s/^(.*)\: .*$/$1/;
		if ($writeto =~ s/^\#//) {
			#nothing
		} else {
			$writeto = "$title/$writeto";
		}
		print "Writing to [[$writeto]]\n";
		$pw->edit("$writeto", $wiki, "Importing from Project Gutenberg using an automated script");
		#sleep 5;
	}

}
	
print "done\n";

Bartleby texts

#This script is for importing public domain works at www.bartleby.com
# I create a text file containing the names of chapters, and URLs for their sources.

use strict;
use LWP::UserAgent;
use Encode;
use Perlwikipedia;

my $title="[[The American Language]]"; # or whatever
my $author="H. L. Mencken"; # or whoever
my $before_first = "";
my $chapterbegin = '<\!\-\- BEGIN CHAPTER \-\->';
my $chapterend = '<\!\-\- BOTTOM CHAPTER\/SECTION NAV CODE \-\->';
my $poem = 0; 
my $test = 0;

# ------------------------------------------------------------------------------------------------------------------------

#login
my $pw=Perlwikipedia->new();
#$pw->{debug} = 1;
$pw->{mech}->agent('Firefox/2.0.0.6');
$pw->set_wiki('en.wikisource.org', 'w');
print "Logging in as Polbot\n";
my $login_status=$pw->login('Polbot','[Password]');
die "I can't log in." unless $login_status == 0;

open(infile, "<chapters.txt");

my $previous_line;
my $current_line = $before_first;
my $next_line = <infile>;

while ($next_line) {	
	#cycle
	$previous_line = $current_line;
	$current_line = $next_line;
	$next_line = <infile>;
	chomp $next_line;
	print "$current_line\n";
	my $footnoted = 0;
	
	next if $current_line =~ /^\#/;
	
	#see how to show them
	$previous_line =~ /^(\[\[.*\]\])/;
	my $previous_show = $1;
	$next_line =~ /^(\[\[.*\]\])/;
	my $next_show = $1;
	if (!$next_line) { $next_show = "" };
	$current_line =~ /^\[\[(?:[^\|]*\|)?(.*)\]\](.*)$/;
	my $current_show = $1;
	my $url = $2;
	$current_line =~ /^\[\[([^\|]*)(\|.*)?\]\]/;
	my $article_name = $1;
	
	print "Article=[[$article_name]]. Show=$current_show. Prev=$previous_show. Next=$next_show. url=$url\n\n";
	
	# write header.
	my $wiki_out .= "{{header2\n| title=$title\n|author=$author\n";
	$wiki_out .= "|section=$current_show\n|previous=$previous_show\n|next=$next_show\n";
	$wiki_out .= "|notes=\n}}\n";
	$wiki_out .= "<poem>\n" if $poem;
	
	# get chapter.
	print "Reading chapter. . .\n";
	my $ua = LWP::UserAgent->new;
	$ua->agent("Mozilla/6.0");
	my $res = $ua->get($url);
	die "Could not connect\n" unless $res->is_success;
	my $work = $res->content;
	
	# Excerpt just the chapters
	$work =~ s/^.*?($chapterbegin.*?)$chapterend.*$/$1/si or die "chapter not excerpted.";
	$work =~ s/&\#160;|&nbsp;/ /gi;	# take out fixed spaces
	$work =~ s/<\!--.*?-->//gi; # remove html comments
	$work =~ s/<TD(\s+VALIGN\s*=\s*"?TOP"?\s*|\s+ALIGN\s*=\s*"?RIGHT"?)+>.*?<\/TD>//ig; # Remove page numbers

	# Fix	footnotes   
	# <A HREF="/185/44.html#note95">95</A>
	# <A NAME="note95"><B>Note 95.</B></A>  <I>Cf.</I> Vogue Affixes in Present-Day Word-Coinage [<A HREF="/185/44.html#txt95">back</A>]</TD></TR></TABLE>

	while ($work =~ /<A HREF=".*?#(note(\d|\.)+)">\d+<\/A>/gi) {
		my $footnote_name = $1;
		$footnoted = 1;

		$work =~ s/^(.*)<A HREF=".*?\#$footnote_name">\d+<\/A>(.*)<A NAME="$footnote_name"><B>Note \d+\.\s*<\/B><\/A>\s*(.*?) \[<A .*?>back<\/A>\](.*)$/$1<ref>$3<\/ref>$2$4/si;
	}
	
	$work =~ s/<font size="?\+1"?><B>(.*?)<\/B><\/font>/==$1==/gi; # wikiformat headers
	$work =~ s/\n?<CENTER><B><A.*?>(.*?)<\/A><\/B><\/CENTER>\n?/\n==$1==\n/gi; # more headers
	$work =~ s/<i>(.*?)<\/i>/''$1''/gi; # wikiformat italics
	$work =~ s/<b>(.*?)<\/b>/'''$1'''/gi; # wikiformat bold
	
	# Wikiformat interior tables
	while ($work =~ /<TABLE WIDTH=\"45\%\" CELLPADDING=3 CELLSPACING=1 BORDER=\"0\">\n?(.*?)\n?<\/TABLE>/is) {
		my $intable = $1;
		$intable =~ s/\n?<tr([^>]*)>\n?/\n|- $1\n/gi;
		$intable =~ s/\n?<td([^>]*)>\n?/\n| $1|/gi;

		$work =~ s/\n?<TABLE WIDTH=\"45\%\" CELLPADDING=3 CELLSPACING=1 BORDER=\"0\">\n?(.*?)\n?<\/TABLE>\n?/\n{|\n$intable\n|}\n/is;
	}
	
	$work =~ s/<\/?T[DR].*?>//gi; # remove table tags
	$work =~ s/<\/?TABLE.*?>//gi; # ditto
	$work =~ s/<\/?FONT.*?>//gi; # remove font tags
	$work =~ s/<a .*?>(.*?)<\/a>/$1/gi; # remove links
	$work =~ s/<[hb]r>//gi; # remove lines and hard returns
	$work =~ s/<\/?dl>//gi; # remove other stuff
	$work =~ s/<img .*?>/\n[image not included]\n/gi; # remove images
	$work =~ s/<\!--.*?-->//gi; # remove html comments
	$work =~ s/<\/?FORM>//gi; # remove forms
	$work =~ s/<center><center>/<center>/gi; #fix bug
	$work =~ s/^\s+/\n/gm if !$poem; # remove initial spaces for prose
	$work =~ s/^:\s*$//gm; # remove lone colons
	$work =~ s/<PAGE NUM=".*?">//gi; # remove page numbers
	$work =~ s/\s*\n\s*\n\s+/\n\n/g; # remove excess whitespace

	$wiki_out .= "$work\n";
	$wiki_out .= "</poem>\n" if $poem;
	
	if ($footnoted) {
		$wiki_out .= "\n==Notes==\n" . '<div style="-moz-column-count: 2; column-count: 2;">' . "<references /></div>\n";
	}
	
	if ($test) {
		open(outfile, ">test.txt");
		print outfile $wiki_out;
		die "done";
	}
	
	#write to Wikipedia.
	print "Writing [[$article_name]].\n";
	$pw->edit($article_name, $wiki_out, "Importing a work from a website, using a perl script.");
	sleep 1;
}

close(infile);

CCEL texts

# This script is for works at the Christian Classics Ethereal Library at www.ccel.org
# Again, it must be modified and tested for each work.

use strict;
use LWP::UserAgent;
use Encode;
use Perlwikipedia;

my $file = shift;
my $start = shift;
my $pw;
my $title = "";
my $author = "";

my $usechaptersfor = "Book [IXVLC]+|On the Making of Man";
my $dontcount = "Prologue|Introduction";
my $Whatwecallsections = "Chapter";

my @articlenames = ();
my @articletitles = ();
my @ranks = ();
my @wikis = ();

open(infile, "<$file") or die "File not found\n";

# Get header (for title and author)
print "Reading header. . .\n";
while (my $line = <infile>) {
	chop $line;
	
	if ($line =~ /<DC\.Title>[A-Z0-9]+\.\s+(.*)<\/DC\.Title>/) {
		$title = $1;
	}
	if ($line =~ /<DC\.Creator sub=\"Author\" scheme=\"short\-form\">(.*)<\/DC\.Creator>/) {
		$author = $1;
	}
	last if $line =~ /<ThML\.body>/;
}
$title = "Nicene and Post-Nicene Fathers: Series II/Volume V";
$author = "[[Author:Philip Schaff|Philip Schaff]] et al.";

my $line;
my $wiki;
my $articletitle = "";
my $rank = "";

until ($articletitle) {
	$line = <infile>;
	if ($line =~ /<div(\d)(?: type=\"[^\"]*\")?(?: n=\"[^\"]*\")? title=\"(.*?)\"/) {
		$rank = $1;
		$articletitle = $2;
		$articletitle =~ s/^(.*?)\&\#8212;.*$/$1/;
		$articletitle =~ s/\&\#230;/ae/g;
		$articletitle =~ s/\&\#198;/AE/g;
		$articletitle =~ s/\.$//;
	}
}

while ($line = <infile>) {
	chop $line;
	# <div3 type="Book" n="I" title="Book I." 
	if ($line =~ /<div(\d)(?: type=\"[^\"]*\")?(?: n=\"[^\"]*\")? title=\"(.*?)\"/) {
		my $temprank = "$1";
		my $temptitle = "$2";
		$temptitle =~ s/^(.*?)\&\#8212;.*$/$1/;
		$temptitle =~ s/\&\#230;/ae/g;
		$temptitle =~ s/\&\#198;/AE/g;
		$temptitle =~ s/\&\#339;/oe/g;
		$temptitle =~ s/\&\#8217;/'/g;
		$temptitle =~ s/\&\#235;/e/g;
		$temptitle =~ s/\.$//;
		
		push @articletitles, $articletitle;
		push @articlenames, "";
		push @ranks, $rank;
		push @wikis, $wiki;
		
		$articletitle = $temptitle;
		$rank = $temprank;
		$wiki = "";
	} else {
		$wiki .= "$line\n";		
	}
}
close(infile);

push @articletitles, $articletitle;
push @articlenames, "";
push @ranks, $rank;
push @wikis, $wiki;

open(outfile, ">test.txt");
print outfile "Contents:\n\n";

print scalar(@articletitles) . " pages.\n";

my $div1 = "";
my $div2 = "";
my $div3 = "";
my $div4 = "";
my $container1 = 0;
my $container2 = 0;
my $container3 = 0;
my $container4 = 0;
my $chaptercount2 = 0;
my $chaptercount3 = 0;
my $chaptercount4 = 0;
my $chaptercount5 = 0;

for my $i(0..$#articletitles) {
	my $articletitle = $articletitles[$i];
	my $articlename = "";
	
	if ($ranks[$i] eq "1") {
		$articlename = "$articletitle";
		print outfile "* [[$title/$articlename|$articletitle]]\n";
		$div1 = $articletitle;
		$container1 = $i;
		
		$chaptercount2 = ($articletitle =~ /^($usechaptersfor)$/) ? 1 : 0;
	} elsif ($ranks[$i] eq "2") {
		if ($chaptercount2 and ($articletitle !~ /^($dontcount)$/i)) {
			$articletitle = "$Whatwecallsections $chaptercount2";
			$articletitles[$i] = "$Whatwecallsections $chaptercount2";
			$chaptercount2++;
		} 
		
		$articlename = "$div1/$articletitle";
		print outfile "** [[$title/$articlename|$articletitle]]\n";
		$div2 = $articletitle;
		$container2 = $i;
		
		if ($wikis[$container1] !~ /== Contents ==/) {
			$wikis[$container1] .= "\n== Contents ==\n";
		}
		$wikis[$container1] .= "* [[$title/$articlename|$articletitle]]\n";
		
		$chaptercount3 = ($articletitle =~ /^($usechaptersfor)$/) ? 1 : 0;
	} elsif ($ranks[$i] eq "3") {
		if ($chaptercount3 and ($articletitle !~ /^($dontcount)$/i)) {
			$articletitle = "$Whatwecallsections $chaptercount3";
			$articletitles[$i] = "$Whatwecallsections $chaptercount3";
			$chaptercount3++;
		} 

		$chaptercount4 = ($articletitle =~ /^($usechaptersfor)$/) ? 1 : 0;
		
		$articlename = "$div1/$div2/$articletitle";
		print outfile "*** [[$title/$articlename|$articletitle]]\n" if $start =~ /test/;
		$div3 = $articletitle;
		$container3 = $i;
		
		if ($wikis[$container2] !~ /== Contents ==/) {
			$wikis[$container2] .= "\n== Contents ==\n";
		}
		$wikis[$container2] .= "* [[$title/$articlename|$articletitle]]\n";
		
	} elsif ($ranks[$i] eq "4") {
		if ($chaptercount4 and ($articletitle !~ /^($dontcount)$/i)) {
			$articletitle = "$Whatwecallsections $chaptercount4";
			$articletitles[$i] = "$Whatwecallsections $chaptercount4";
			$chaptercount4++;
		}

		$chaptercount5 = ($articletitle =~ /^($usechaptersfor)$/) ? 1 : 0;

		$articlename = "$div1/$div2/$div3/$articletitle";
		print outfile "**** [[$title/$articlename|$articletitle]]\n" if $start =~ /test/;
		$div4 = $articletitle;
		$container4 = $i;
		
		if ($wikis[$container3] !~ /== Contents ==/) {
			$wikis[$container3] .= "\n== Contents ==\n";
		}
		$wikis[$container3] .= "* [[$title/$articlename|$articletitle]]\n";
		
	} else { # 5
		if ($chaptercount5 and ($articletitle !~ /^($dontcount)$/i)) {
			$articletitle = "$Whatwecallsections $chaptercount5";
			$articletitles[$i] = "$Whatwecallsections $chaptercount5";
			$chaptercount5++;
		}

		$articlename = "$div1/$div2/$div3/$div4/$articletitle";
		print outfile "***** [[$title/$articlename|$articletitle]]\n" if $start =~ /test/;
		if ($wikis[$container4] !~ /== Contents ==/) {
			$wikis[$container4] .= "\n== Contents ==\n";
		}
		$wikis[$container4] .= "* [[$title/$articlename|$articletitle]]\n";
	}	
	
	$articlenames[$i] = $articlename;
}

close (outfile);		
die "Wrote TOC" if $start =~ /test/;
print "Wrote TOC\n\n";
sleep 5;

# login
my $pw=Perlwikipedia->new();
#$pw->{debug} = 1;
$pw->{mech}->agent('Firefox/2.0.0.6');
$pw->set_wiki('en.wikisource.org', 'w');
print "Logging in as Polbot\n";
my $login_status=$pw->login('Polbot','[password]');
die "I can't log in." unless $login_status == 0;

for my $i($start..$#articletitles) {
	my $articletitle = $articletitles[$i];
	my $articlename = $articlenames[$i];
	my $wiki = $wikis[$i];
	print "$i: $articletitle\n";

	#change <h2>, <i>, bbsc, and refs
	#$wiki =~ s/<h\d[^>]*>(.*?)<\/h\d>/'''{{bbsc|$1}}'''\n/gis; # Make headers into smallcaps, just for this one
	$wiki =~ s/<span class=\"c\d\d[^>]*>(.*?)<\/span>/{{small-caps|$1}}/gs; #make c24 span into smallcaps, just for this one
	$wiki =~ s/<h\d[^>]*>(.*?)<\/h\d>/'''$1'''\n/gis; # Make headers into bold
	$wiki =~ s/<\/?(i|em)>/\'\'/gi; # change italics
	$wiki =~ s/<span class=\"(?:upper|sc)\"[^>]*>(.*?)<\/span>/{{small-caps|$1}}/gs; # bbsc
	$wiki =~ s/<note[^>]*>(.*?)<\/note>/<ref>$1<\/ref>/gs; # refs
	$wiki =~ s/<\/l>/<br>/g;               # poem lines
	$wiki =~ s/<p( [^>]*)?>/\n/gi;  # p to hard return
	
	#take out p, index, </div>, hr, verse, l, a, etc.
	#keep br, ref
	$wiki =~ s/<\!--.*?-->//gi;      # html comments
	$wiki =~ s/<hr[^>]*>//gi;        # hr 	
	$wiki =~ s/<\/?p( [^>]*)?>//gi;  # p
	$wiki =~ s/<\/?pb( [^>]*)?>//gi; # pb
	$wiki =~ s/<\/?index[^>]*>//gi;  # index
	$wiki =~ s/<\/?div[^>]*>//gi;    # div	
	$wiki =~ s/<\/?scrip[^>]*>//gs;  # scripRef
	$wiki =~ s/<\/?span( [^>]*)?>//gi;    # span
	$wiki =~ s/<verse [^>]*>//g;     # poem start
	$wiki =~ s/<\/verse>//g;         # poem end
	$wiki =~ s/<l [^>]*>//gi;        # l
	$wiki =~ s/<\/?a[^>]*>//gs;      # a
	$wiki =~ s/<\/?name( [^>]*)?>//gi;  # name

	$wiki =~ s/^ +//gm;              # Take out initial spaces
	$wiki =~ s/\n\s*\n\s*\n/\n\n/g;  # Take out excess whitespace	
	#$wiki .= "\n==Footnotes==\n" . '<div style="-moz-column-count: 2; column-count: 2;">' . "<references /></div>" if $wiki =~ /<ref>/;
	$wiki .= "\n==Footnotes==\n<references /></div>" if $wiki =~ /<ref>/;

	my $showtitle = linkslashes("$title/$articlename");
	$showtitle =~ /(.*)\/(.*)/;
	my $showsection = $2;
	$showtitle=$1;
	
	my $header = "{{header2\n|title=$showtitle\n|author=$author\n";
	$header .= "|section=$showsection\n";
	if ($i > 0) {
		$header .= "|previous=[[$title/" . $articlenames[$i-1] . "|" . $articletitles[$i-1] . "]]\n";
	} else {
		$header .= "|previous=\n";
	}
	if ($articlenames[$i+1]) {
		$header .= "|next=[[$title/" . $articlenames[$i+1] . "|" . $articletitles[$i+1] . "]]\n|notes=\n}}\n";
	} else {
		$header .= "|next=\n|notes=\n}}\n";
	}
	
	$wiki = $header . $wiki;
	
	print "Writing to [[$title/$articlename]]\n";

	$pw->edit("$title/$articlename", $wiki, "Importing from Christian Classics Etherial Library, using an automated script");
	#sleep 5;

}

sub linkslashes {
	my $line = shift;
	my $analyzeline = $line;
	
	while ($analyzeline =~ s/^([^\[\]\|]*)\/([^\[\]\|]*)\/(.*)$/$1\/\[\[$1\/$2\|$2\]\]\/$3/) {};
	$analyzeline =~ s/^([^\/]*)\//[[$1]]\//;
	
	return $analyzeline;
}