Attachment #162410 for bug #198877

View | Details | Raw Unified | Return to bug 198877
Collapse All | Expand All





PORTNAME=	portscout
PORTVERSION=	0.8.1
PORTREVISION=	4
CATEGORIES=	ports-mgmt
MASTER_SITES=	http://mirror.inerd.com/FreeBSD/distfiles/${PORTNAME}/ \
		http://www.atarininja.org/~wxs/distfiles/ \
		http://www.inerd.com/software/${PORTNAME}/

MAINTAINER=	shaun@FreeBSD.org
COMMENT=	Tool to scan for new versions of FreeBSD ports

LICENSE=	BSD2CLAUSE
SQLITE3_DESC=	Use SQLite backend instead of PostgreSQL

RUN_DEPENDS=	p5-DBI>=0:${PORTSDIR}/databases/p5-DBI \
		p5-Proc-Queue>=0:${PORTSDIR}/devel/p5-Proc-Queue \
		p5-Net>=0:${PORTSDIR}/net/p5-Net \
		p5-URI>=0:${PORTSDIR}/net/p5-URI \
		p5-XML-XPath>=0:${PORTSDIR}/textproc/p5-XML-XPath \
		p5-MIME-Lite>=0:${PORTSDIR}/mail/p5-MIME-Lite \
		p5-libwww>=0:${PORTSDIR}/www/p5-libwww \
		p5-JSON>0:${PORTSDIR}/converters/p5-JSON

OPTIONS_DEFINE=		DOCS
OPTIONS_DEFAULT=	PGSQL

OPTIONS_MULTI=		DATABASE
OPTIONS_MULTI_DATABASE=	SQLITE PGSQL

DATABASE_DESC=	Database Backends

USES=		perl5 shebangfix
NO_BUILD=	yes
SHEBANG_FILES=	${WRKSRC}/portscout.pl


PORTDOCS=	UPDATING portscout-portconfig.txt xml-datasrc-example.xml

SQLITE_USE=		SQLITE=3
SQLITE_RUN_DEPENDS=	p5-DBD-SQLite>=0:${PORTSDIR}/databases/p5-DBD-SQLite
		p5-Net>=0:${PORTSDIR}/net/p5-Net			\
		p5-URI>=0:${PORTSDIR}/net/p5-URI			\
		p5-XML-XPath>=0:${PORTSDIR}/textproc/p5-XML-XPath	\
		p5-MIME-Lite>=0:${PORTSDIR}/mail/p5-MIME-Lite		\
		p5-libwww>=0:${PORTSDIR}/www/p5-libwww

PGSQL_USES=		pgsql
PGSQL_RUN_DEPENDS=	p5-DBD-Pg>=0:${PORTSDIR}/databases/p5-DBD-Pg

.include <bsd.port.options.mk>

.if ${PORT_OPTIONS:MSQLITE3}
#USE_SQLITE=	3
#RUN_DEPENDS+=	p5-DBD-SQLite>=0:${PORTSDIR}/databases/p5-DBD-SQLite
.elif !defined(WITHOUT_PGSQL)
#USES+=		pgsql
#RUN_DEPENDS+=	p5-DBD-Pg>=0:${PORTSDIR}/databases/p5-DBD-Pg
.endif

pre-everything::
.if ${PORT_OPTIONS:MSQLITE}
	@${ECHO_MSG} "+-------------------------------------------------------------+"
	@${ECHO_MSG} "| Warning! although SQLite is supported, portscout will only  |"
	@${ECHO_MSG} "| operate in non-forking mode with this database backend. It  |"

	@${ECHO_MSG} "+-------------------------------------------------------------+"
.endif

post-extract:
	@${CP} ${FILESDIR}/files-Portscout-SiteHandler-GitHub.pm ${WRKSRC}/Portscout/SiteHandler/GitHub.pm
	@${CP} ${FILESDIR}/files-Portscout-SiteHandler-PyPI.pm ${WRKSRC}/Portscout/SiteHandler/PyPI.pm

post-patch:
.if ${PORT_OPTIONS:MSQLITE}
	@${REINPLACE_CMD} 's/^\([^#]*DBI:Pg.*\)$$/#\1/g' ${WRKSRC}/portscout.conf
	@${REINPLACE_CMD} 's/^#\(.*DBI:SQLite.*\)$$/\1/g' ${WRKSRC}/portscout.conf
.endif

test:
	${PERL} ${WRKSRC}/t/00-use.t
	${PERL} ${WRKSRC}/t/01-vercompare.t
.if ${PORT_OPTIONS:MSQLITE}
	${PERL} ${WRKSRC}/t/10-sqlite.t
#.else
#	${PERL} ${WRKSRC}/10-postgresql.t




#------------------------------------------------------------------------------
# Copyright (C) 2014, Jasper Lievisse Adriaanse <jasper@openbsd.org>
#
# Permission to use, copy, modify, and distribute this software for any
# purpose with or without fee is hereby granted, provided that the above
# copyright notice and this permission notice appear in all copies.
#
# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
#
#------------------------------------------------------------------------------

package Portscout::SiteHandler::GitHub;

use JSON qw(decode_json);
use LWP::UserAgent;

use Portscout::Const;
use Portscout::Config;

use strict;

require 5.006;


#------------------------------------------------------------------------------
# Globals
#------------------------------------------------------------------------------

push @Portscout::SiteHandler::sitehandlers, __PACKAGE__;

our %settings;


#------------------------------------------------------------------------------
# Func: new()
# Desc: Constructor.
#
# Args: n/a
#
# Retn: $self
#------------------------------------------------------------------------------

sub new
{
	my $self      = {};
	my $class     = shift;

	$self->{name} = 'GitHub';

	bless ($self, $class);
	return $self;
}


#------------------------------------------------------------------------------
# Func: CanHandle()
# Desc: Ask if this handler (package) can handle the given site.
#
# Args: $url - URL of site.
#
# Retn: $res - true/false.
#------------------------------------------------------------------------------

sub CanHandle
{
	my $self = shift;

	my ($url) = @_;

	return ($url =~ /^https?:\/\/([^\/.]+\.)?github\.com\/(.*?)\/tar.gz/);
}


#------------------------------------------------------------------------------
# Func: GetFiles()
# Desc: Extract a list of files from the given URL. In the case of GitHub,
#       we are actually pulling the files from the project's Atom feed and
#       extract the release url, containing the tag it was based on.
#
# Args: $url     - URL we would normally fetch from.
#       \%port   - Port hash fetched from database.
#       \@files  - Array to put files into.
#
# Retn: $success - False if file list could not be constructed; else, true.
#------------------------------------------------------------------------------

sub GetFiles
{
	my $self = shift;

	my ($url, $port, $files) = @_;
	my $projname;

	if ($url =~ /https:\/\/github\.com\/(.*?)\/archive\//) {
	    $projname = $1;
	} elsif ($url =~ /https:\/\/github.com\/downloads\/(.*)\//) {
	    $projname = $1;
	}

	if ($projname) {
		my ($query, $ua, $response, $items, $json);

		# First check if there's a latest releases endpoint
		$query = 'https://api.github.com/repos/' . $projname . '/releases/latest';

		_debug("GET $query");
		$ua = LWP::UserAgent->new;
		$ua->agent(USER_AGENT);
		$ua->timeout($settings{http_timeout});

		$response = $ua->request(HTTP::Request->new(GET => $query));

		if (!$response->is_success || $response->status_line !~ /^2/) {
			_debug('GET failed: ' . $response->status_line);
			# Project didn't do any releases, so let's try tags instead.
			$query = 'https://api.github.com/repos/' . $projname . '/tags';
			_debug("GET $query");
			$ua = LWP::UserAgent->new;
			$ua->agent(USER_AGENT);
			$ua->timeout($settings{http_timeout});

			$response = $ua->request(HTTP::Request->new(GET => $query));

			if (!$response->is_success || $response->status_line !~ /^2/) {
			    _debug('GET failed: ' . $response->status_line);
			    return 0;
			}

			$json = decode_json($response->decoded_content);
			foreach my $tag (@$json) {
			    my $tag_url = $tag->{tarball_url};
			    push(@$files, $tag_url);
			}

			_debug('Found ' . scalar @$files . ' files');
			return 1;
		}

		$json = decode_json($response->decoded_content);
		push(@$files, $json->{tarball_url});

		_debug('Found ' . scalar @$files . ' files');
	} else {
		return 0;
	}

	return 1;
}


#------------------------------------------------------------------------------
# Func: _debug()
# Desc: Print a debug message.
#
# Args: $msg - Message.
#
# Retn: n/a
#------------------------------------------------------------------------------

sub _debug
{
	my ($msg) = @_;

	$msg = '' if (!$msg);

	print STDERR "(" . __PACKAGE__ . ") $msg\n" if ($settings{debug});
}

1;




#------------------------------------------------------------------------------
# Copyright (C) 2015, Jasper Lievisse Adriaanse <jasper@openbsd.org>
#
# Permission to use, copy, modify, and distribute this software for any
# purpose with or without fee is hereby granted, provided that the above
# copyright notice and this permission notice appear in all copies.
#
# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
#
#------------------------------------------------------------------------------

package Portscout::SiteHandler::PyPI;

use JSON qw(decode_json);
use LWP::UserAgent;

use Portscout::Const;
use Portscout::Config;

use strict;

require 5.006;


#------------------------------------------------------------------------------
# Globals
#------------------------------------------------------------------------------

push @Portscout::SiteHandler::sitehandlers, __PACKAGE__;

our %settings;


#------------------------------------------------------------------------------
# Func: new()
# Desc: Constructor.
#
# Args: n/a
#
# Retn: $self
#------------------------------------------------------------------------------

sub new
{
	my $self      = {};
	my $class     = shift;

	$self->{name} = 'PyPI';

	bless ($self, $class);
	return $self;
}


#------------------------------------------------------------------------------
# Func: CanHandle()
# Desc: Ask if this handler (package) can handle the given site.
#
# Args: $url - URL of site.
#
# Retn: $res - true/false.
#------------------------------------------------------------------------------

sub CanHandle
{
	my $self = shift;

	my ($url) = @_;

	return ($url =~ /https?:\/\/pypi\.python\.org\//);
}


#------------------------------------------------------------------------------
# Func: GetFiles()
# Desc: Extract a list of files from the given URL. Simply query the API.
#
# Args: $url     - URL we would normally fetch from.
#       \%port   - Port hash fetched from database.
#       \@files  - Array to put files into.
#
# Retn: $success - False if file list could not be constructed; else, true.
#------------------------------------------------------------------------------

sub GetFiles
{
	my $self = shift;

	my ($url, $port, $files) = @_;

	my ($pypi, $package, $resp, $query, $ua);
	$pypi = 'https://pypi.python.org/pypi/';

	# Strip all the digits at the end to keep the stem of the module.
	if ($port->{distname} =~ /(.*?)-(\d+)/) {
	    $package = $1;
	}

	$query = $pypi . $package . '/json';

	_debug("GET $query");
	$ua = LWP::UserAgent->new;
	$ua->agent(USER_AGENT);
	$resp = $ua->request(HTTP::Request->new(GET => $query));
	if ($resp->is_success) {
	    my ($json, $info, $version);

    	    $json = decode_json($resp->decoded_content);
	    $info = $json->{info};
	    $version = $info->{version};
	    next unless $version;

	    push(@$files, $json->{releases}{$version}[0]{filename});
	} else {
	    _debug("GET failed: " . $resp->code);
	    return 0;
	}

	return 1;
}


#------------------------------------------------------------------------------
# Func: _debug()
# Desc: Print a debug message.
#
# Args: $msg - Message.
#
# Retn: n/a
#------------------------------------------------------------------------------

sub _debug
{
	my ($msg) = @_;

	$msg = '' if (!$msg);

	print STDERR "(" . __PACKAGE__ . ") $msg\n" if ($settings{debug});
}

1;




--- Portscout/DataSrc/Ports.pm.orig	2011-04-09 17:19:03 UTC
+++ Portscout/DataSrc/Ports.pm
@@ -201,7 +201,7 @@ sub BuildDB
 
 	my $lastbuild = getstat('buildtime', TYPE_INT);
 
-	print "Looking for updated ports...\n\n"
+	print "Incremental build: Looking for updated ports...\n\n"
 		if ($incremental);
 
 	$got_ports = 0;
@@ -243,12 +243,15 @@ sub BuildDB
 			my (@fields, $maintainer, $port);
 
 			@fields = split /\|/;
-			$maintainer = $fields[5];
+			$maintainer = lc($fields[5]);
 			$port = $fields[1];
 			$port =~ s/^(?:.*\/)?([^\/]+)\/([^\/]+)$/$1\/$2/;
 
-			$portsmaintok{$port} = $maintainer
-				if ($maintainers{$maintainer});
+			if ($maintainers{$maintainer}) {
+				$portsmaintok{$port} = $maintainer;
+				print "Maintainer match: $maintainer $port \n"
+					unless ($settings{quiet});
+			}
 		}
 
 		close $if;
@@ -264,7 +267,7 @@ sub BuildDB
 
 		opendir my $catdir, $settings{ports_dir}."/$cat";
 
-		print "Scanning $cat...\n"
+		print "Scanning $cat ...\n"
 			unless ($settings{quiet});
 
 		while (my $name = readdir $catdir) {
@@ -276,9 +279,9 @@ sub BuildDB
 			# port directory's mtime; skip if not updated.
 			if ($incremental) {
 				my ($updated);
-
 				opendir my $portdir, $settings{ports_dir}."/$cat/$name";
-
+				print "Scanning $cat/$name ... "
+					unless ($settings{debug});
 				while (my $subfile = readdir $portdir) {
 					my ($subfile_path, $fi);
 
@@ -289,12 +292,18 @@ sub BuildDB
 						or die "Couldn't stat $subfile_path: $!";
 
 					if ($fi->mtime > $lastbuild) {
+						print "$subfile (mtime: $fi->mtime) modified updated since last build: $lastbuild \n"
+							unless ($settings{debug});
 						$updated = 1;
 						last;
 					}
 				}
 
-				next if (!$updated);
+				if (!$updated) {
+					print "Not modified since last build: $lastbuild \n"
+						unless ($settings{debug});
+					next;
+				}
 			}
 
 			# Check this port is wanted by user
@@ -307,7 +316,7 @@ sub BuildDB
 					&& $settings{indexfile_enable}) {
 				next if (!$portsmaintok{"$cat/$name"});
 			}
-
+			print "Matched: " . $cat/$name . "\n";
 			push @ports, "$cat/$name";
 		}
 	}

Line 0 Link Here

(-)ports-mgmt/portscout/files/patch-Portscout_SiteHandler.pm (+11 lines)
	1	--- Portscout/SiteHandler.pm.orig 2010-04-29 01:07:51 UTC
	2	+++ Portscout/SiteHandler.pm
	3	@@ -31,6 +31,8 @@ package Portscout::SiteHandler;
	4	use XML::XPath;
	5	use XML::XPath::XMLParser;
	6
	7	+use Portscout::SiteHandler::GitHub;
	8	+use Portscout::SiteHandler::PyPI;
	9	use Portscout::SiteHandler::SourceForge;
	10
	11	use strict;




--- Portscout/SiteHandler/GitHub.pm.orig	2015-10-24 04:02:57 UTC
+++ Portscout/SiteHandler/GitHub.pm
@@ -97,7 +97,9 @@ sub GetFiles
 	my ($url, $port, $files) = @_;
 	my $projname;
 
-	if ($url =~ /https:\/\/github\.com\/(.*?)\/archive\//) {
+	if ($url =~ /https?:\/\/codeload\.github\.com\/(.*?)\/tar.gz\//) {
+	    $projname = $1;
+	} elsif ($url =~ /https:\/\/github\.com\/(.*?)\/archive\//) {
 	    $projname = $1;
 	} elsif ($url =~ /https:\/\/github.com\/downloads\/(.*)\//) {
 	    $projname = $1;

Line 0 Link Here

(-)ports-mgmt/portscout/files/patch-Portscout_SiteHandler_PyPI.pm (+11 lines)
	1	--- Portscout/SiteHandler/PyPI.pm.orig 2015-10-24 04:03:07 UTC
	2	+++ Portscout/SiteHandler/PyPI.pm
	3	@@ -115,7 +115,7 @@ sub GetFiles
	4	$info = $json->{info};
	5	$version = $info->{version};
	6	next unless $version;
	7	-
	8	+ _debug("GET success: " . $resp->code . " Filename: " . $json->{releases}{$version}[0]{filename});
	9	push(@$files, $json->{releases}{$version}[0]{filename});
	10	} else {
	11	_debug("GET failed: " . $resp->code);




--- portscout.pl.orig	2015-10-24 04:02:57 UTC
+++ portscout.pl
@@ -463,7 +463,7 @@ sub VersionCheck
 
 		$i++;
 
-		info($k, 'Checking site: ' . strchop($site, 60));
+		info($k, 'Checking site: ' . strchop($site, 200));
 
 		# Look to see if the URL contains the distfile version.
 		# This will affect our checks and guesses later on.
@@ -493,6 +493,7 @@ sub VersionCheck
 		}
 
 		# Check for special handler for this site first
+		print "Checking for site handler for $site \n";
 		if (my $sh = Portscout::SiteHandler->FindHandler($site))
 		{
 			info($k, $site, 'Using dedicated site handler for site.');
@@ -699,7 +700,7 @@ sub VersionCheck
 				# We keep a counter of "lies" from each site, and only
 				# re-check every so often.
 
-				if ($sitedata->{liecount} > 0) {
+				if ($sitedata->{liecount} > 0 && $settings{skip_liars}) {
 					info($k, $site, 'Not doing any guessing; site has previously lied.');
 					$sths->{sitedata_decliecount}->execute($sitedata->{host})
 						unless($settings{precious_data});
@@ -713,7 +714,8 @@ sub VersionCheck
 
 				# Got a response which wasn't HTTP 4xx -> bail out
 				if ($response->is_success && $response->status_line !~ /^4/) {
-					info($k, $site, 'Not doing any guessing; site is lieing to us.');
+					print "URL: $url\n";
+					info($k, $site, "Skip guessing: Response not 4xx to a file that shouldnt exist (". $response->status_line .")");
 					$sths->{sitedata_initliecount}->execute($sitedata->{host})
 						unless($settings{precious_data});
 					next;
@@ -797,7 +799,7 @@ sub VersionCheck
 							$new_found = 1;
 							last;
 						} else {
-							info($k, $site, "Guess failed $port->{ver} -> $guess_v");
+							info($k, $site, "Guess failed $port->{ver} -> $guess_v (". $response->status_line .")");
 						}
 
 						last if ($new_found);

Lines 14-32 Link Here

(-)ports-mgmt/portscout/pkg-plist (-10 / +12 lines)
14	%%DATADIR%%/sql/sqlite_init.sql	14	%%DATADIR%%/sql/sqlite_init.sql
15	%%DATADIR%%/sql/sqlite_destroy.sql	15	%%DATADIR%%/sql/sqlite_destroy.sql
16	%%DATADIR%%/sql/sqlite_upgrade_0.8_to_0.8.1.sql	16	%%DATADIR%%/sql/sqlite_upgrade_0.8_to_0.8.1.sql
17	%%SITE_PERL%%/Portscout/SQL/SQLite.pm	17	%%SITE_PERL%%/Portscout.pm
18	%%SITE_PERL%%/Portscout/SQL/Pg.pm	18	%%SITE_PERL%%/Portscout/API.pm
19	%%SITE_PERL%%/Portscout/SiteHandler/SourceForge.pm	19	%%SITE_PERL%%/Portscout/Config.pm
		20	%%SITE_PERL%%/Portscout/Const.pm
		21	%%SITE_PERL%%/Portscout/DataSrc.pm
20	%%SITE_PERL%%/Portscout/DataSrc/Ports.pm	22	%%SITE_PERL%%/Portscout/DataSrc/Ports.pm
21	%%SITE_PERL%%/Portscout/DataSrc/XML.pm	23	%%SITE_PERL%%/Portscout/DataSrc/XML.pm
		24	%%SITE_PERL%%/Portscout/Make.pm
		25	%%SITE_PERL%%/Portscout/SQL.pm
		26	%%SITE_PERL%%/Portscout/SQL/Pg.pm
		27	%%SITE_PERL%%/Portscout/SQL/SQLite.pm
22	%%SITE_PERL%%/Portscout/SiteHandler.pm	28	%%SITE_PERL%%/Portscout/SiteHandler.pm
23	%%SITE_PERL%%/Portscout/SQL.pm	29	%%SITE_PERL%%/Portscout/SiteHandler/GitHub.pm
24	%%SITE_PERL%%/Portscout/Make.pm	30	%%SITE_PERL%%/Portscout/SiteHandler/PyPI.pm
25	%%SITE_PERL%%/Portscout/Const.pm	31	%%SITE_PERL%%/Portscout/SiteHandler/SourceForge.pm
26	%%SITE_PERL%%/Portscout/DataSrc.pm
27	%%SITE_PERL%%/Portscout/Template.pm	32	%%SITE_PERL%%/Portscout/Template.pm
28	%%SITE_PERL%%/Portscout/Util.pm	33	%%SITE_PERL%%/Portscout/Util.pm
29	%%SITE_PERL%%/Portscout/API.pm
30	%%SITE_PERL%%/Portscout/Config.pm
31	%%SITE_PERL%%/Portscout.pm
32	%%SITE_PERL%%/portscout.pod	34	%%SITE_PERL%%/portscout.pod

Return to bug 198877

Lines 3-20 Link Here

(-)ports-mgmt/portscout/Makefile (-23 / +33 lines)
3		3
4	PORTNAME= portscout	4	PORTNAME= portscout
5	PORTVERSION= 0.8.1	5	PORTVERSION= 0.8.1
6	PORTREVISION= 3	6	PORTREVISION= 4
7	CATEGORIES= ports-mgmt	7	CATEGORIES= ports-mgmt
8	MASTER_SITES= http://mirror.inerd.com/FreeBSD/distfiles/${PORTNAME}/ \	8	MASTER_SITES= http://mirror.inerd.com/FreeBSD/distfiles/${PORTNAME}/ \
9	http://www.atarininja.org/~wxs/distfiles/ \	9	http://www.atarininja.org/~wxs/distfiles/ \
10	http://www.inerd.com/software/${PORTNAME}/	10	http://www.inerd.com/software/${PORTNAME}/
11		11
12	MAINTAINER= shaun@FreeBSD.org	12	MAINTAINER= shaun@FreeBSD.org
13	COMMENT= Tool to scan for new versions of FreeBSD ports	13	COMMENT= Tool to scan for new versions of FreeBSD ports
14		14
15	OPTIONS_DEFINE= SQLITE3 DOCS	15	LICENSE= BSD2CLAUSE
16	SQLITE3_DESC= Use SQLite backend instead of PostgreSQL
17		16
		17	RUN_DEPENDS= p5-DBI>=0:${PORTSDIR}/databases/p5-DBI \
		18	p5-Proc-Queue>=0:${PORTSDIR}/devel/p5-Proc-Queue \
		19	p5-Net>=0:${PORTSDIR}/net/p5-Net \
		20	p5-URI>=0:${PORTSDIR}/net/p5-URI \
		21	p5-XML-XPath>=0:${PORTSDIR}/textproc/p5-XML-XPath \
		22	p5-MIME-Lite>=0:${PORTSDIR}/mail/p5-MIME-Lite \
		23	p5-libwww>=0:${PORTSDIR}/www/p5-libwww \
		24	p5-JSON>0:${PORTSDIR}/converters/p5-JSON
		25
		26	OPTIONS_DEFINE= DOCS
		27	OPTIONS_DEFAULT= PGSQL
		28
		29	OPTIONS_MULTI= DATABASE
		30	OPTIONS_MULTI_DATABASE= SQLITE PGSQL
		31
		32	DATABASE_DESC= Database Backends
		33
18	USES= perl5 shebangfix	34	USES= perl5 shebangfix
19	NO_BUILD= yes	35	NO_BUILD= yes
20	SHEBANG_FILES= ${WRKSRC}/portscout.pl	36	SHEBANG_FILES= ${WRKSRC}/portscout.pl
Lines 21-46 Link Here
21		37
22	PORTDOCS= UPDATING portscout-portconfig.txt xml-datasrc-example.xml	38	PORTDOCS= UPDATING portscout-portconfig.txt xml-datasrc-example.xml
23		39
24	RUN_DEPENDS= p5-DBI>=0:${PORTSDIR}/databases/p5-DBI \	40	SQLITE_USE= SQLITE=3
25	p5-Proc-Queue>=0:${PORTSDIR}/devel/p5-Proc-Queue \	41	SQLITE_RUN_DEPENDS= p5-DBD-SQLite>=0:${PORTSDIR}/databases/p5-DBD-SQLite
26	p5-Net>=0:${PORTSDIR}/net/p5-Net \
27	p5-URI>=0:${PORTSDIR}/net/p5-URI \
28	p5-XML-XPath>=0:${PORTSDIR}/textproc/p5-XML-XPath \
29	p5-MIME-Lite>=0:${PORTSDIR}/mail/p5-MIME-Lite \
30	p5-libwww>=0:${PORTSDIR}/www/p5-libwww
31		42
		43	PGSQL_USES= pgsql
		44	PGSQL_RUN_DEPENDS= p5-DBD-Pg>=0:${PORTSDIR}/databases/p5-DBD-Pg
		45
32	.include <bsd.port.options.mk>	46	.include <bsd.port.options.mk>
33		47
34	.if ${PORT_OPTIONS:MSQLITE3}
35	#USE_SQLITE= 3
36	#RUN_DEPENDS+= p5-DBD-SQLite>=0:${PORTSDIR}/databases/p5-DBD-SQLite
37	.elif !defined(WITHOUT_PGSQL)
38	#USES+= pgsql
39	#RUN_DEPENDS+= p5-DBD-Pg>=0:${PORTSDIR}/databases/p5-DBD-Pg
40	.endif
41
42	pre-everything::	48	pre-everything::
43	.if ${PORT_OPTIONS:MSQLITE3}	49	.if ${PORT_OPTIONS:MSQLITE}
44	@${ECHO_MSG} "+-------------------------------------------------------------+"	50	@${ECHO_MSG} "+-------------------------------------------------------------+"
45	@${ECHO_MSG} "\| Warning! although SQLite is supported, portscout will only \|"	51	@${ECHO_MSG} "\| Warning! although SQLite is supported, portscout will only \|"
46	@${ECHO_MSG} "\| operate in non-forking mode with this database backend. It \|"	52	@${ECHO_MSG} "\| operate in non-forking mode with this database backend. It \|"
Lines 48-55 Link Here
48	@${ECHO_MSG} "+-------------------------------------------------------------+"	54	@${ECHO_MSG} "+-------------------------------------------------------------+"
49	.endif	55	.endif
50		56
		57	post-extract:
		58	@${CP} ${FILESDIR}/files-Portscout-SiteHandler-GitHub.pm ${WRKSRC}/Portscout/SiteHandler/GitHub.pm
		59	@${CP} ${FILESDIR}/files-Portscout-SiteHandler-PyPI.pm ${WRKSRC}/Portscout/SiteHandler/PyPI.pm
		60
51	post-patch:	61	post-patch:
52	.if ${PORT_OPTIONS:MSQLITE3}	62	.if ${PORT_OPTIONS:MSQLITE}
53	@${REINPLACE_CMD} 's/^\([^#]DBI:Pg.\)$$/#\1/g' ${WRKSRC}/portscout.conf	63	@${REINPLACE_CMD} 's/^\([^#]DBI:Pg.\)$$/#\1/g' ${WRKSRC}/portscout.conf
54	@${REINPLACE_CMD} 's/^#\(.DBI:SQLite.\)$$/\1/g' ${WRKSRC}/portscout.conf	64	@${REINPLACE_CMD} 's/^#\(.DBI:SQLite.\)$$/\1/g' ${WRKSRC}/portscout.conf
55	.endif	65	.endif
Lines 96-102 Link Here
96	test:	106	test:
97	${PERL} ${WRKSRC}/t/00-use.t	107	${PERL} ${WRKSRC}/t/00-use.t
98	${PERL} ${WRKSRC}/t/01-vercompare.t	108	${PERL} ${WRKSRC}/t/01-vercompare.t
99	.if ${PORT_OPTIONS:MSQLITE3}	109	.if ${PORT_OPTIONS:MSQLITE}
100	${PERL} ${WRKSRC}/t/10-sqlite.t	110	${PERL} ${WRKSRC}/t/10-sqlite.t
101	#.else	111	#.else
102	# ${PERL} ${WRKSRC}/10-postgresql.t	112	# ${PERL} ${WRKSRC}/10-postgresql.t

Line 0 Link Here

(-)ports-mgmt/portscout/files/files-Portscout-SiteHandler-GitHub.pm (+175 lines)
		1	#------------------------------------------------------------------------------
		2	# Copyright (C) 2014, Jasper Lievisse Adriaanse <jasper@openbsd.org>
		3	#
		4	# Permission to use, copy, modify, and distribute this software for any
		5	# purpose with or without fee is hereby granted, provided that the above
		6	# copyright notice and this permission notice appear in all copies.
		7	#
		8	# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
		9	# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
		10	# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
		11	# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
		12	# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
		13	# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
		14	# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
		15	#
		16	#------------------------------------------------------------------------------
		17
		18	package Portscout::SiteHandler::GitHub;
		19
		20	use JSON qw(decode_json);
		21	use LWP::UserAgent;
		22
		23	use Portscout::Const;
		24	use Portscout::Config;
		25
		26	use strict;
		27
		28	require 5.006;
		29
		30
		31	#------------------------------------------------------------------------------
		32	# Globals
		33	#------------------------------------------------------------------------------
		34
		35	push @Portscout::SiteHandler::sitehandlers, __PACKAGE__;
		36
		37	our %settings;
		38
		39
		40	#------------------------------------------------------------------------------
		41	# Func: new()
		42	# Desc: Constructor.
		43	#
		44	# Args: n/a
		45	#
		46	# Retn: $self
		47	#------------------------------------------------------------------------------
		48
		49	sub new
		50	{
		51	my $self = {};
		52	my $class = shift;
		53
		54	$self->{name} = 'GitHub';
		55
		56	bless ($self, $class);
		57	return $self;
		58	}
		59
		60
		61	#------------------------------------------------------------------------------
		62	# Func: CanHandle()
		63	# Desc: Ask if this handler (package) can handle the given site.
		64	#
65	# Args: $url - URL of site.
66	#
67	# Retn: $res - true/false.
68	#------------------------------------------------------------------------------
69
70	sub CanHandle
71	{
72	my $self = shift;
73
74	my ($url) = @_;
75
76	return ($url =~ /^https?:\/\/([^\/.]+\.)?github\.com\/(.*?)\/tar.gz/);
77	}
78
79
80	#------------------------------------------------------------------------------
81	# Func: GetFiles()
82	# Desc: Extract a list of files from the given URL. In the case of GitHub,
83	# we are actually pulling the files from the project's Atom feed and
84	# extract the release url, containing the tag it was based on.
85	#
86	# Args: $url - URL we would normally fetch from.
87	# \%port - Port hash fetched from database.
88	# \@files - Array to put files into.
89	#
90	# Retn: $success - False if file list could not be constructed; else, true.
91	#------------------------------------------------------------------------------
92
93	sub GetFiles
94	{
95	my $self = shift;
96
97	my ($url, $port, $files) = @_;
98	my $projname;
99
100	if ($url =~ /https:\/\/github\.com\/(.*?)\/archive\//) {
101	$projname = $1;
102	} elsif ($url =~ /https:\/\/github.com\/downloads\/(.*)\//) {
103	$projname = $1;
104	}
105
106	if ($projname) {
107	my ($query, $ua, $response, $items, $json);
108
109	# First check if there's a latest releases endpoint
110	$query = 'https://api.github.com/repos/' . $projname . '/releases/latest';
111
112	_debug("GET $query");
113	$ua = LWP::UserAgent->new;
114	$ua->agent(USER_AGENT);
115	$ua->timeout($settings{http_timeout});
116
117	$response = $ua->request(HTTP::Request->new(GET => $query));
118
119	if (!$response->is_success \|\| $response->status_line !~ /^2/) {
120	_debug('GET failed: ' . $response->status_line);
121	# Project didn't do any releases, so let's try tags instead.
122	$query = 'https://api.github.com/repos/' . $projname . '/tags';
123	_debug("GET $query");
124	$ua = LWP::UserAgent->new;
125	$ua->agent(USER_AGENT);
126	$ua->timeout($settings{http_timeout});
127
128	$response = $ua->request(HTTP::Request->new(GET => $query));
129
130	if (!$response->is_success \|\| $response->status_line !~ /^2/) {
131	_debug('GET failed: ' . $response->status_line);
132	return 0;
133	}
134
135	$json = decode_json($response->decoded_content);
136	foreach my $tag (@$json) {
137	my $tag_url = $tag->{tarball_url};
138	push(@$files, $tag_url);
139	}
140
141	_debug('Found ' . scalar @$files . ' files');
142	return 1;
143	}
144
145	$json = decode_json($response->decoded_content);
146	push(@$files, $json->{tarball_url});
147
148	_debug('Found ' . scalar @$files . ' files');
149	} else {
150	return 0;
151	}
152
153	return 1;
154	}
155
156
157	#------------------------------------------------------------------------------
158	# Func: _debug()
159	# Desc: Print a debug message.
160	#
161	# Args: $msg - Message.
162	#
163	# Retn: n/a
164	#------------------------------------------------------------------------------
165
166	sub _debug
167	{
168	my ($msg) = @_;
169
170	$msg = '' if (!$msg);
171
172	print STDERR "(" . __PACKAGE__ . ") $msg\n" if ($settings{debug});
173	}
174
175	1;

Line 0 Link Here

(-)ports-mgmt/portscout/files/files-Portscout-SiteHandler-PyPI.pm (+147 lines)
		1	#------------------------------------------------------------------------------
		2	# Copyright (C) 2015, Jasper Lievisse Adriaanse <jasper@openbsd.org>
		3	#
		4	# Permission to use, copy, modify, and distribute this software for any
		5	# purpose with or without fee is hereby granted, provided that the above
		6	# copyright notice and this permission notice appear in all copies.
		7	#
		8	# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
		9	# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
		10	# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
		11	# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
		12	# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
		13	# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
		14	# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
		15	#
		16	#------------------------------------------------------------------------------
		17
		18	package Portscout::SiteHandler::PyPI;
		19
		20	use JSON qw(decode_json);
		21	use LWP::UserAgent;
		22
		23	use Portscout::Const;
		24	use Portscout::Config;
		25
		26	use strict;
		27
		28	require 5.006;
		29
		30
		31	#------------------------------------------------------------------------------
		32	# Globals
		33	#------------------------------------------------------------------------------
		34
		35	push @Portscout::SiteHandler::sitehandlers, __PACKAGE__;
		36
		37	our %settings;
		38
		39
		40	#------------------------------------------------------------------------------
		41	# Func: new()
		42	# Desc: Constructor.
		43	#
		44	# Args: n/a
		45	#
		46	# Retn: $self
		47	#------------------------------------------------------------------------------
		48
		49	sub new
		50	{
		51	my $self = {};
		52	my $class = shift;
		53
		54	$self->{name} = 'PyPI';
		55
		56	bless ($self, $class);
		57	return $self;
		58	}
		59
		60
		61	#------------------------------------------------------------------------------
		62	# Func: CanHandle()
		63	# Desc: Ask if this handler (package) can handle the given site.
		64	#
65	# Args: $url - URL of site.
66	#
67	# Retn: $res - true/false.
68	#------------------------------------------------------------------------------
69
70	sub CanHandle
71	{
72	my $self = shift;
73
74	my ($url) = @_;
75
76	return ($url =~ /https?:\/\/pypi\.python\.org\//);
77	}
78
79
80	#------------------------------------------------------------------------------
81	# Func: GetFiles()
82	# Desc: Extract a list of files from the given URL. Simply query the API.
83	#
84	# Args: $url - URL we would normally fetch from.
85	# \%port - Port hash fetched from database.
86	# \@files - Array to put files into.
87	#
88	# Retn: $success - False if file list could not be constructed; else, true.
89	#------------------------------------------------------------------------------
90
91	sub GetFiles
92	{
93	my $self = shift;
94
95	my ($url, $port, $files) = @_;
96
97	my ($pypi, $package, $resp, $query, $ua);
98	$pypi = 'https://pypi.python.org/pypi/';
99
100	# Strip all the digits at the end to keep the stem of the module.
101	if ($port->{distname} =~ /(.*?)-(\d+)/) {
102	$package = $1;
103	}
104
105	$query = $pypi . $package . '/json';
106
107	_debug("GET $query");
108	$ua = LWP::UserAgent->new;
109	$ua->agent(USER_AGENT);
110	$resp = $ua->request(HTTP::Request->new(GET => $query));
111	if ($resp->is_success) {
112	my ($json, $info, $version);
113
114	$json = decode_json($resp->decoded_content);
115	$info = $json->{info};
116	$version = $info->{version};
117	next unless $version;
118
119	push(@$files, $json->{releases}{$version}[0]{filename});
120	} else {
121	_debug("GET failed: " . $resp->code);
122	return 0;
123	}
124
125	return 1;
126	}
127
128
129	#------------------------------------------------------------------------------
130	# Func: _debug()
131	# Desc: Print a debug message.
132	#
133	# Args: $msg - Message.
134	#
135	# Retn: n/a
136	#------------------------------------------------------------------------------
137
138	sub _debug
139	{
140	my ($msg) = @_;
141
142	$msg = '' if (!$msg);
143
144	print STDERR "(" . __PACKAGE__ . ") $msg\n" if ($settings{debug});
145	}
146
147	1;

Line 0 Link Here

(-)ports-mgmt/portscout/files/patch-Portscout_DataSrc_Ports.pm (+80 lines)
		1	--- Portscout/DataSrc/Ports.pm.orig 2011-04-09 17:19:03 UTC
		2	+++ Portscout/DataSrc/Ports.pm
		3	@@ -201,7 +201,7 @@ sub BuildDB
		4
		5	my $lastbuild = getstat('buildtime', TYPE_INT);
		6
		7	- print "Looking for updated ports...\n\n"
		8	+ print "Incremental build: Looking for updated ports...\n\n"
		9	if ($incremental);
		10
		11	$got_ports = 0;
		12	@@ -243,12 +243,15 @@ sub BuildDB
		13	my (@fields, $maintainer, $port);
		14
		15	@fields = split /\\|/;
		16	- $maintainer = $fields[5];
		17	+ $maintainer = lc($fields[5]);
		18	$port = $fields[1];
		19	$port =~ s/^(?:.*\/)?([^\/]+)\/([^\/]+)$/$1\/$2/;
		20
		21	- $portsmaintok{$port} = $maintainer
		22	- if ($maintainers{$maintainer});
		23	+ if ($maintainers{$maintainer}) {
		24	+ $portsmaintok{$port} = $maintainer;
		25	+ print "Maintainer match: $maintainer $port \n"
		26	+ unless ($settings{quiet});
		27	+ }
		28	}
		29
		30	close $if;
		31	@@ -264,7 +267,7 @@ sub BuildDB
		32
		33	opendir my $catdir, $settings{ports_dir}."/$cat";
		34
		35	- print "Scanning $cat...\n"
		36	+ print "Scanning $cat ...\n"
		37	unless ($settings{quiet});
		38
		39	while (my $name = readdir $catdir) {
		40	@@ -276,9 +279,9 @@ sub BuildDB
		41	# port directory's mtime; skip if not updated.
		42	if ($incremental) {
		43	my ($updated);
		44	-
		45	opendir my $portdir, $settings{ports_dir}."/$cat/$name";
		46	-
		47	+ print "Scanning $cat/$name ... "
		48	+ unless ($settings{debug});
		49	while (my $subfile = readdir $portdir) {
		50	my ($subfile_path, $fi);
		51
		52	@@ -289,12 +292,18 @@ sub BuildDB
		53	or die "Couldn't stat $subfile_path: $!";
		54
		55	if ($fi->mtime > $lastbuild) {
		56	+ print "$subfile (mtime: $fi->mtime) modified updated since last build: $lastbuild \n"
		57	+ unless ($settings{debug});
		58	$updated = 1;
		59	last;
		60	}
		61	}
		62
		63	- next if (!$updated);
		64	+ if (!$updated) {
65	+ print "Not modified since last build: $lastbuild \n"
66	+ unless ($settings{debug});
67	+ next;
68	+ }
69	}
70
71	# Check this port is wanted by user
72	@@ -307,7 +316,7 @@ sub BuildDB
73	&& $settings{indexfile_enable}) {
74	next if (!$portsmaintok{"$cat/$name"});
75	}
76	-
77	+ print "Matched: " . $cat/$name . "\n";
78	push @ports, "$cat/$name";
79	}
80	}

Line 0 Link Here

(-)ports-mgmt/portscout/files/patch-Portscout_SiteHandler_GitHub.pm (+13 lines)
	1	--- Portscout/SiteHandler/GitHub.pm.orig 2015-10-24 04:02:57 UTC
	2	+++ Portscout/SiteHandler/GitHub.pm
	3	@@ -97,7 +97,9 @@ sub GetFiles
	4	my ($url, $port, $files) = @_;
	5	my $projname;
	6
	7	- if ($url =~ /https:\/\/github\.com\/(.*?)\/archive\//) {
	8	+ if ($url =~ /https?:\/\/codeload\.github\.com\/(.*?)\/tar.gz\//) {
	9	+ $projname = $1;
	10	+ } elsif ($url =~ /https:\/\/github\.com\/(.*?)\/archive\//) {
	11	$projname = $1;
	12	} elsif ($url =~ /https:\/\/github.com\/downloads\/(.*)\//) {
	13	$projname = $1;

Line 0 Link Here

(-)ports-mgmt/portscout/files/patch-portscout.pl (+47 lines)
	1	--- portscout.pl.orig 2015-10-24 04:02:57 UTC
	2	+++ portscout.pl
	3	@@ -463,7 +463,7 @@ sub VersionCheck
	4
	5	$i++;
	6
	7	- info($k, 'Checking site: ' . strchop($site, 60));
	8	+ info($k, 'Checking site: ' . strchop($site, 200));
	9
	10	# Look to see if the URL contains the distfile version.
	11	# This will affect our checks and guesses later on.
	12	@@ -493,6 +493,7 @@ sub VersionCheck
	13	}
	14
	15	# Check for special handler for this site first
	16	+ print "Checking for site handler for $site \n";
	17	if (my $sh = Portscout::SiteHandler->FindHandler($site))
	18	{
	19	info($k, $site, 'Using dedicated site handler for site.');
	20	@@ -699,7 +700,7 @@ sub VersionCheck
	21	# We keep a counter of "lies" from each site, and only
	22	# re-check every so often.
	23
	24	- if ($sitedata->{liecount} > 0) {
	25	+ if ($sitedata->{liecount} > 0 && $settings{skip_liars}) {
	26	info($k, $site, 'Not doing any guessing; site has previously lied.');
	27	$sths->{sitedata_decliecount}->execute($sitedata->{host})
	28	unless($settings{precious_data});
	29	@@ -713,7 +714,8 @@ sub VersionCheck
	30
	31	# Got a response which wasn't HTTP 4xx -> bail out
	32	if ($response->is_success && $response->status_line !~ /^4/) {
	33	- info($k, $site, 'Not doing any guessing; site is lieing to us.');
	34	+ print "URL: $url\n";
	35	+ info($k, $site, "Skip guessing: Response not 4xx to a file that shouldnt exist (". $response->status_line .")");
	36	$sths->{sitedata_initliecount}->execute($sitedata->{host})
	37	unless($settings{precious_data});
	38	next;
	39	@@ -797,7 +799,7 @@ sub VersionCheck
	40	$new_found = 1;
	41	last;
	42	} else {
	43	- info($k, $site, "Guess failed $port->{ver} -> $guess_v");
	44	+ info($k, $site, "Guess failed $port->{ver} -> $guess_v (". $response->status_line .")");
	45	}
	46
	47	last if ($new_found);