SFGrab

#!/usr/bin/perl -w
use strict;
use POSIX qw(ceil floor);
 
use HTTP::Lite;
use HTML::Strip;
 
my $keyword = "ada";
my $showDescription = "no";
my $debug = "no";
 
my $stripper = HTML::Strip->new();
 
sub makeRequest {
	my ($url, $mode) = @_;
	my $http = new HTTP::Lite;
	eval {
		my $rq = $http->request($url) or die;
	};
	if (!$@) {
		my $body = $http->body();
		my $cleanBody = "";
		if ($mode eq "mode=raw") {
			$cleanBody = $body;
		}
		if ($mode eq "mode=clean") {
			$cleanBody = $stripper->parse($body);
		}
		my @split = split(/\n/, $cleanBody);
		for (my $run=0;$run<(@split);$run++) {
			$split[$run] =~ s/^\s+|\s+$//g; # whitespace and linebreak removal
			$split[$run] =~ s/\t//g; # tab removal
		}
		my @cleanAr;
		for (my $run=0;$run<(@split);$run++) {
			if ($split[$run] ne "") {
				push @cleanAr, $split[$run];
				print "$run: ".$split[$run]."\n" if ($debug eq "yes");
			}
		}
		$stripper->eof;
		return @cleanAr;
	} else {
		warn("meh! request failed");
	}
	$http = undef;
}
 
sub url {
	my $offset = shift;
	return "http://sourceforge.net/search/?words=$keyword&sort=score&sortdir=desc&offset=$offset";
}
 
my $offset = 0;
 
my @clAr = &makeRequest(&url($offset), "mode=clean");
 
my $entryCount = $clAr[6];
my $requestCount = ceil($entryCount / 10);
 
for (my $run=0;$run<$requestCount;$run++) {
	my @ar = &makeRequest(&url($offset), "mode=clean");
	my @raw = &makeRequest(&url($offset), "mode=raw");
 
	my @unixnames;
	my @projectNames;
	my @descriptions;
	for (my $irun=0;$irun<(@raw);$irun++) {
		# unixnames and project names
		if (index($raw[$irun], "/projects/") != -1) {
			my @nSplit = split(/\/projects\//, $raw[$irun]);
			my @fSplit = split(/\/">/, $nSplit[1]);
			my @titleSplit = split(/<\/a>/, $fSplit[1]);
			push @projectNames, $titleSplit[0];
			push @unixnames, $fSplit[0];
		}
		# description
		if (index($raw[$irun], "class=\"description\"") != -1) {
			my $descContent = "";
			for (my $iirun=$irun;(index($raw[$iirun], "<p>")==-1);$iirun++) {
				$descContent .= $raw[$iirun];
			}
			$descContent = $stripper->parse($descContent);
			$stripper->eof;
			print $descContent if ($debug eq "yes");
			push @descriptions, $descContent;
		}
	}
 
	# print the information
	for (my $irun=0;$irun<10;$irun++) {
		print $projectNames[$irun]." http://".$unixnames[$irun].".sf.net\n";
		print "---\n";
		print $descriptions[$irun]."\n\n";
	}
 
	$offset += 10;
 
	@ar = undef;
}