[GR-5380] Implement mx_pkgtests.findtop100.

PullRequest: fastr/1078

com.oracle.truffle.r.test.packages/r/install.packages.R

@@ -114,6 +114,7 @@ usage <- function() {
 					  "[--install-dependents-first]",
 					  "[--run-mode mode]",
 					  "[--pkg-filelist file]",
+					  "[--find-top100]",
 					  "[--run-tests]",
 					  "[--testdir dir]",
 					  "[--print-ok-installs]",
@@ -645,7 +646,7 @@ do.it <- function() {
 		for (pkgname in test.pkgnames) {
 			pkg <- toinstall.pkgs[pkgname, ]
 			# pretend we are accessing CRAN if list.canonical
-			list.contriburl = ifelse(list.canonical, "https://cran.r-project.org/src/contrib", contriburl)
+			list.contriburl = ifelse(list.canonical, "https://cran.r-project.org/src/contrib", pkg["Repository"])
 			cat(pkg["Package"], pkg["Version"], paste0(list.contriburl, "/", pkgname, "_", pkg["Version"], ".tar.gz"), "\n", sep=",")
 		}
 	}
@@ -887,6 +888,8 @@ parse.args <- function() {
 			use.installed.pkgs <<- TRUE
 		} else if (a == "--invert-pkgset") {
 			invert.pkgset <<- TRUE
+		} else if (a == "--find-top100") {
+			find.top100 <<- TRUE
 		} else {
 			if (grepl("^-.*", a)) {
 				usage()
@@ -965,8 +968,27 @@ get.initial.package.blacklist <- function() {
 	}
 }
 
+do.find.top100 <- function() {
+	avail.pkgs <- available.packages(type="source");
+	install.packages('cranlogs')
+	library('cranlogs')
+	top100 <- cran_top_downloads(when = c("last-day", "last-week", "last-month"), count = 100)
+	names <- top100[['package']]
+	l = length(names)
+	for (i in 1:l) {
+		pkgname <- names[[i]]
+		pkg <- avail.pkgs[pkgname, ]
+		list.contriburl = ifelse(list.canonical, "https://cran.r-project.org/src/contrib", pkg["Repository"])
+		cat(pkg["Package"], pkg["Version"], paste0(list.contriburl, "/", pkgname, "_", pkg["Version"], ".tar.gz"), "\n", sep=",")
+		cat(pkgname)
+		if (i != l) {
+			cat(',')
+		}
+	}
+	cat('\n')
+}
+
 run.setup <- function() {
-	parse.args()
 	check.libs()
 	check.pkgfilelist()
 	set.repos()
@@ -977,8 +999,14 @@ run.setup <- function() {
 }
 
 run <- function() {
-	run.setup()
-    do.it()
+	parse.args()
+	if (find.top100) {
+		set.repos()
+		do.find.top100()
+	} else {
+	    run.setup()
+        do.it()
+    }
 }
 
 quiet <- F
@@ -1013,6 +1041,7 @@ gnur <- FALSE
 list.versions <- FALSE
 list.canonical <- FALSE
 invert.pkgset <- F
+find.top100 <- F
 
 if (!interactive()) {
     run()

com.oracle.truffle.r.test.packages/top100

+R6
 Rcpp
+curl
+jsonlite
+tibble
 ggplot2
-plyr
+dplyr
+rlang
 stringr
-colorspace
+stringi
 digest
+magrittr
+plyr
+BH
 reshape2
-RColorBrewer
+pkgconfig
+DBI
 scales
-manipulate
+lazyeval
+backports
+RColorBrewer
+colorspace
+glue
 mime
-labeling
-munsell
-proto
+htmltools
+bindrcpp
+evaluate
+plogr
 gtable
+munsell
 dichromat
-zoo
-RCurl
-bitops
-evaluate
-car
-jsonlite
+labeling
 knitr
-formatR
-highr
-R6
+yaml
+bindr
+data.table
+memoise
+assertthat
+git2r
 httr
-lme4
-rJava
-quantreg
-SparseM
-nloptr
-dplyr
-DBI
-minqa
-MASS
 markdown
-lattice
+htmlwidgets
+openssl
+base64enc
+readr
+bitops
+XML
 devtools
+highr
+rJava
+rmarkdown
+httpuv
+tidyr
+caTools
+car
+shiny
+hms
+igraph
 Formula
-pbkrtest
-gtools
+RcppArmadillo
+gridExtra
+zoo
+xtable
+rprojroot
+sandwich
+viridisLite
+TTR
+checkmate
+lme4
 RcppEigen
-caTools
-magrittr
-yaml
-htmltools
+sourcetools
+quantmod
 Hmisc
-mgcv
+purrr
+viridis
 forecast
-Matrix
-lazyeval
-BH
-sandwich
-mvtnorm
-assertthat
-latticeExtra
-rjson
-rpart
-foreach
-XML
-roxygen2
-rstudioapi
-iterators
-xtable
-RJSONIO
-quadprog
+lubridate
+bit64
+SparseM
+readxl
 sp
-acepack
-RcppArmadillo
-e1071
-igraph
-tseries
-memoise
+quantreg
 gdata
-tm
-rmarkdown
-brew
-shiny
-timeDate
-codetools
-chron
-randomForest
-data.table
-testthat
-slam
-nnet
-xlsx
+xts
+withr
+RSQLite
+rstudioapi
+tseries
+psych
+RCurl
 whisker
-rgl
-boot
-reshape
-NLP
-gridExtra
-mnormt
-maps
-gplots
-survival
-foreign
-httpuv
+bit
+agricolae
+nloptr
+minqa
+blob
+pbkrtest
+MatrixModels
+cellranger
+mvtnorm

mx.fastr/mx_fastr.py

@@ -501,6 +501,8 @@ _commands = {
     'pkgtest' : [mx_fastr_pkgs.pkgtest, ['options']],
     'pkgtest-cmp' : [mx_fastr_pkgs.pkgtest_cmp, ['gnur_path fastr_path']],
     'r-pkgtest-analyze' : [mx_fastr_pkgs.pta, ['options']],
+    'r-findtop100' : [mx_fastr_pkgs.find_top100, ['options']],
+    'r-duppkgs' : [mx_fastr_pkgs.remove_dup_pkgs, ['options']],
     'installpkgs' : [mx_fastr_pkgs.installpkgs, '[options]'],
     'mkgramrd': [mx_fastr_mkgramrd.mkgramrd, '[options]'],
     'rcopylib' : [mx_copylib.copylib, '[]'],

mx.fastr/mx_fastr_pkgs.py

@@ -558,3 +558,20 @@ def pkgtest_cmp(args):
     with open(args[1]) as f:
         fastr_content = f.readlines()
     return _fuzzy_compare(gnur_content, fastr_content)
+
+def find_top100(args):
+    libinstall = join(_fastr_suite_dir(), "top100.tmp")
+    if not os.path.exists(libinstall):
+        os.mkdir(libinstall)
+    os.environ['R_LIBS_USER'] = libinstall
+    _installpkgs(['--find-top100', '--use-installed-pkgs'])
+
+def remove_dup_pkgs(args):
+    pkgs = args[0].split(",")
+    x = dict()
+    for p in pkgs:
+        x[p] = 1
+    result = []
+    for p in x.iterkeys():
+        result += p
+    return result