PHRAS-3823 Report - command line report generator (#4265)

* command report connections

* some report

* prefix beta

* Update .dockerignore

* some fix, add range option

---------

Co-authored-by: Nicolas Maillat <maillat@alchemy.fr>
This commit is contained in:
Aina Sitraka
2023-03-11 18:40:27 +03:00
committed by GitHub
parent 1f8b9928b1
commit e7b280a70c
13 changed files with 678 additions and 10 deletions

View File

@@ -19,6 +19,7 @@
!/bin/developer
!/bin/setup
!/bin/maintenance
!/bin/report
/cache
/config/configuration.yml
/config/configuration-compiled.php

1
.gitignore vendored
View File

@@ -51,6 +51,7 @@
!/bin/developer
!/bin/setup
!/bin/maintenance
!/bin/report
# Exclude composer
composer.phar

32
bin/report Executable file
View File

@@ -0,0 +1,32 @@
#!/usr/bin/env php
<?php
use Alchemy\Phrasea\CLI;
use Alchemy\Phrasea\Core\Version;
use Alchemy\Phrasea\Command\Report\ConnectionsCommand;
use Alchemy\Phrasea\Command\Report\CountAssetsCommand;
use Alchemy\Phrasea\Command\Report\CountUsersCommand;
require_once __DIR__ . '/../lib/autoload.php';
$version = new Version();
$cli = new CLI("
__ _ _ _
/__\ ___ _ __ ___ _ __| |_ | |_ ___ ___ | |___
/ \/// _ \ '_ \ / _ \| '__| __| | __/ _ \ / _ \| / __|
/ _ \ __/ |_) | (_) | | | |_ | || (_) | (_) | \__ \
\/ \_/\___| .__/ \___/|_| \__| \__\___/ \___/|_|___/
|_|
Phraseanet Copyright (C) 2004 Alchemy
This program comes with ABSOLUTELY NO WARRANTY.
This is free software, and you are welcome to redistribute it
under certain conditions; type `about:license` for details.\n\n".
' Report Tools ,'. $version->getName() . ' ' . $version->getNumber());
$cli->command(new ConnectionsCommand());
$cli->command(new CountAssetsCommand());
$cli->command(new CountUsersCommand());
$cli->run();

View File

@@ -0,0 +1,175 @@
<?php
namespace Alchemy\Phrasea\Command\Report;
use Alchemy\Phrasea\Application\Helper\NotifierAware;
use Alchemy\Phrasea\Command\Command;
use Alchemy\Phrasea\Core\LazyLocator;
use Alchemy\Phrasea\Notification\Attachment;
use Alchemy\Phrasea\Notification\Mail\MailReportConnections;
use Alchemy\Phrasea\Notification\Receiver;
use Alchemy\Phrasea\Report\Report;
use Cocur\Slugify\Slugify;
use Symfony\Component\Console\Input\InputInterface;
use Symfony\Component\Console\Input\InputOption;
use Symfony\Component\Console\Output\OutputInterface;
use Symfony\Component\HttpKernel\Exception\NotFoundHttpException;
abstract class AbstractReportCommand extends Command
{
use NotifierAware;
protected $sbasId;
protected $emails;
protected $dmin;
protected $dmax;
protected $type;
protected $range;
protected $isAppboxConnection = false;
public function __construct($name)
{
parent::__construct($name);
$this
->addOption('databox_id', null, InputOption::VALUE_REQUIRED, 'the application databox')
->addOption('email', null, InputOption::VALUE_REQUIRED | InputOption::VALUE_IS_ARRAY ,'emails to send the report')
->addOption('dmin', null, InputOption::VALUE_REQUIRED, 'minimum date yyyy-mm-dd')
->addOption('dmax', null, InputOption::VALUE_REQUIRED, 'maximum date yyyy-mm-dd, until today if not set')
->addOption('range', null, InputOption::VALUE_REQUIRED, "period range until now eg: <info>'10 days', '2 weeks', '6 months', ' 1 year'</info>")
;
}
protected function doExecute(InputInterface $input, OutputInterface $output)
{
$this->setDelivererLocator(new LazyLocator($this->container, 'notification.deliverer'));
$this->sbasId = $input->getOption('databox_id');
$this->emails = $input->getOption('email');
$this->dmin = $input->getOption('dmin');
$this->dmax = $input->getOption('dmax');
$this->range = $input->getOption('range');
if(!empty($this->range) && (!empty($this->dmin) || !empty($this->dmax))) {
$output->writeln("<error>do not use '--range' with '--dmin' or '--dmax'</error>");
return 1;
}
if (!empty($this->range)) {
$matches = [];
preg_match("/(\d+) (day|week|month|year)s?/i", $this->range, $matches);
$n = count($matches);
if ($n === 3) {
try {
$this->dmin = (new \DateTime('-' . $matches[0]))->format('Y-m-d');
} catch (\Exception $e) {
// not happen cause don't match if bad format
$output->writeln("<error>invalid value form '--range' option</error>");
return 1;
}
} else {
$output->writeln("<error>invalid value form '--range' option</error>");
return 1;
}
}
if (empty($this->emails)) {
$output->writeln("<error>set '--email' option</error>");
return 1;
}
if (!$this->isDateOk($this->dmin)) {
$output->writeln("<error>invalid value from '--dmin' option</error>");
return 1;
}
if (!empty($this->dmax) && !$this->isDateOk($this->dmax)) {
$output->writeln("<error>invalid value from '--dmax' option</error>");
return 1;
}
$report = $this->getReport($input, $output);
if (!$report instanceof Report) {
return 1;
}
$report->setFormat(Report::FORMAT_CSV);
$absoluteDirectoryPath = \p4string::addEndSlash($this->container['tmp.download.path'])
.'report' . DIRECTORY_SEPARATOR
. date('Ymd');
$suffixFileName = "_" . $this->dmin . "_to_";
$suffixFileName = !empty($this->dmax) ? $suffixFileName . $this->dmax: $suffixFileName . "now";
if ($this->isAppboxConnection) {
$absoluteDirectoryPath .= 'appbox';
} else {
$absoluteDirectoryPath .= 'Sbas' . $this->sbasId;
}
$report->render($absoluteDirectoryPath, $suffixFileName);
$filePath = $absoluteDirectoryPath . DIRECTORY_SEPARATOR . $this->normalizeString($report->getName()).$suffixFileName . '.csv';
$attachement = new Attachment($filePath);
foreach ($this->emails as $email) {
$receiver = new Receiver('', $email);
$mail = MailReportConnections::create($this->container, $receiver);
$this->deliver($mail, false, [$attachement]);
}
$output->writeln("<info>finish !</info>");
return 0;
}
/**
* @param InputInterface $input
* @param OutputInterface $output
* @return Report
*/
abstract protected function getReport(InputInterface $input, OutputInterface $output);
/**
* @param int $sbasId
* @return \databox
*/
protected function findDbOr404($sbasId)
{
$db = $this->container->getApplicationBox()->get_databox(($sbasId));
if(!$db) {
throw new NotFoundHttpException(sprintf('Databox %s not found', $sbasId));
}
return $db;
}
private function isDateOk($date)
{
$matches = [];
preg_match("/(\d{4}-\d{2}-\d{2})/i", $date, $matches);
$n = count($matches);
if ($n === 2) {
return true;
}
return false;
}
private function normalizeString($filename)
{
return (new Slugify())->slugify($filename, '-');
}
}

View File

@@ -0,0 +1,58 @@
<?php
namespace Alchemy\Phrasea\Command\Report;
use Alchemy\Phrasea\Report\ReportConnections;
use Symfony\Component\Console\Input\InputInterface;
use Symfony\Component\Console\Input\InputOption;
use Symfony\Component\Console\Output\OutputInterface;
class ConnectionsCommand extends AbstractReportCommand
{
const TYPES = ['user', 'nav', 'nav,version', 'os', 'os,nav', 'os,nav,version', 'res'];
public function __construct()
{
parent::__construct('connections:all');
$this
->setDescription('BETA - Get all connections report')
->addOption('type', null, InputOption::VALUE_REQUIRED, 'type of report connections, if not defined or empty it is for all connections')
->setHelp(
"eg: bin/report connections:all --databox_id 2 --email 'admin@alchemy.fr' --dmin '2022-12-01' --dmax '2023-01-01' --type 'os,nav' \n"
. "\<TYPE>type of report\n"
. "- <info>'' or not defined </info>all connections\n"
. "- <info>'user' </info> connections by user\n"
. "- <info>'nav' </info> connections by browser\n"
. "- <info>'nav,version' </info> connections by browser, version \n"
. "- <info>'os' </info> connections by OS \n"
. "- <info>'os,nav' </info> connections by OS, browser \n"
. "- <info>'os,nav,version' </info> connections by OS, Browser, Version\n"
. "- <info>'res' </info> connections by Screen Res \n"
);
}
public function getReport(InputInterface $input, OutputInterface $output)
{
$type = $input->getOption('type');
if (!empty($type) && !in_array($type, self::TYPES)) {
$output->writeln("<error>wrong '--type' option (--help for available value)</error>");
return 1;
}
return
(new ReportConnections(
$this->findDbOr404($this->sbasId),
[
'dmin' => $this->dmin,
'dmax' => $this->dmax,
'group' => $type,
'anonymize' => $this->container['conf']->get(['registry', 'modules', 'anonymous-report'])
]
))
->setAppKey($this->container['conf']->get(['main', 'key']));
}
}

View File

@@ -0,0 +1,63 @@
<?php
namespace Alchemy\Phrasea\Command\Report;
use Alchemy\Phrasea\Report\ReportCountAssets;
use Symfony\Component\Console\Input\InputInterface;
use Symfony\Component\Console\Input\InputOption;
use Symfony\Component\Console\Output\OutputInterface;
class CountAssetsCommand extends AbstractReportCommand
{
const TYPES = [
'added,year',
'added,year,month',
'downloaded,year',
'downloaded,year,month',
'downloaded,year,month,action',
'most-downloaded'
];
public function __construct()
{
parent::__construct('count:assets');
$this
->setDescription('BETA - Get assets count')
->addOption('type', null, InputOption::VALUE_REQUIRED, 'type of count assets report ')
->setHelp(
"eg: bin/report count:assets --databox_id 2 --email 'admin@alchemy.fr' --dmin '2021-12-01' --dmax '2023-01-01' --type 'added,year,month' \n"
. "\<TYPE>type of report\n"
. "- <info>'added,year' </info> number of added assets per year\n"
. "- <info>'added,year,month' </info> number of added assets per year, month\n"
. "- <info>'downloaded,year' </info> number of downloaded per year \n"
. "- <info>'downloaded,year,month' </info> number of downloaded per year, month \n"
. "- <info>'downloaded,year,month,action' </info> number of downloaded per year, month, action (direct download or by email) \n"
. "- <info>'most-downloaded' </info> The 10 most downloaded assets \n"
);
}
/**
* @inheritDoc
*/
protected function getReport(InputInterface $input, OutputInterface $output)
{
$type = $input->getOption('type');
if (!empty($type) && !in_array($type, self::TYPES)) {
$output->writeln("<error>wrong '--type' option (--help for available value)</error>");
return 1;
}
return new ReportCountAssets(
$this->findDbOr404($this->sbasId),
[
'dmin' => $this->dmin,
'dmax' => $this->dmax,
'group' => $type
]
);
}
}

View File

@@ -0,0 +1,66 @@
<?php
namespace Alchemy\Phrasea\Command\Report;
use Alchemy\Phrasea\Report\ReportUsers;
use Symfony\Component\Console\Input\InputInterface;
use Symfony\Component\Console\Input\InputOption;
use Symfony\Component\Console\Output\OutputInterface;
use Symfony\Component\HttpKernel\Exception\NotFoundHttpException;
class CountUsersCommand extends AbstractReportCommand
{
const TYPES = [
'added,year',
'added,year,month',
];
public function __construct()
{
parent::__construct('count:users');
$this
->setDescription('BETA - Get users count')
->addOption('type', null, InputOption::VALUE_REQUIRED, 'type of users count report')
->setHelp(
"eg: bin/report count:users --databox_id 2 --email 'admin@alchemy.fr' --dmin '2022-01-01' --dmax '2023-01-01' --type 'added,year,month' \n"
. "\<TYPE>type users count report\n"
. "- <info>'added,year'</info> number of newly added user per year\n"
. "- <info>'added,year,month' </info> number of newly added user per year, month\n"
);
}
/**
* @inheritDoc
*/
protected function getReport(InputInterface $input, OutputInterface $output)
{
$type = $input->getOption('type');
$this->isAppboxConnection = true;
if (!empty($type) && !in_array($type, self::TYPES)) {
$output->writeln("<error>wrong '--type' option (--help for available value)</error>");
return 1;
}
// get just one databox registered to initialize the base class Report
$databoxes = $this->container->getDataboxes();
if (count($databoxes) > 0) {
$databox = current($databoxes);
} else {
throw new NotFoundHttpException("NO databox set on this application");
}
return new ReportUsers(
$databox,
[
'dmin' => $this->dmin,
'dmax' => $this->dmax,
'group' => $type
]
);
}
}

View File

@@ -0,0 +1,37 @@
<?php
namespace Alchemy\Phrasea\Notification\Mail;
class MailReportConnections extends AbstractMail
{
/**
* @inheritDoc
*/
public function getSubject()
{
return $this->app->trans("mail:: report", [], 'messages', $this->getLocale());
}
/**
* @inheritDoc
*/
public function getMessage()
{
return $this->app->trans("mail:: report messages", [], 'messages', $this->getLocale());
}
/**
* @inheritDoc
*/
public function getButtonText()
{
}
/**
* @inheritDoc
*/
public function getButtonURL()
{
}
}

View File

@@ -83,4 +83,9 @@ class Excel
$this->writer->close();
}
public function getWriter()
{
return $this->writer;
}
}

View File

@@ -12,6 +12,7 @@ namespace Alchemy\Phrasea\Report;
use Alchemy\Phrasea\Application;
use Alchemy\Phrasea\Out\Module\Excel;
use Cocur\Slugify\Slugify;
abstract class Report
@@ -105,14 +106,14 @@ abstract class Report
return $this->format;
}
public function render()
public function render($absoluteDirectoryPath = null, $suffixFileName = null)
{
switch($this->format) {
//case self::FORMAT_XLS:
case self::FORMAT_CSV:
case self::FORMAT_ODS:
case self::FORMAT_XLSX:
$this->renderAsExcel();
$this->renderAsExcel($absoluteDirectoryPath, $suffixFileName);
break;
default:
// should not happen since format is checked before
@@ -120,25 +121,40 @@ abstract class Report
}
}
private function renderAsExcel()
private function renderAsExcel($absoluteDirectoryPath = null, $suffixFileName = null)
{
$filename = $this->normalizeString($this->getName()) . $suffixFileName;
switch($this->format) {
//case self::FORMAT_XLS:
// $excel = new Excel(Excel::FORMAT_XLS);
// header('Content-Type: application/vnd.ms-excel');
// break;
case self::FORMAT_XLSX:
$excel = new Excel(Excel::FORMAT_XLSX, $this->getName() . ".xlsx");
$filename .= ".xlsx";
$excel = new Excel(Excel::FORMAT_XLSX, $filename);
break;
case self::FORMAT_ODS:
$excel = new Excel(Excel::FORMAT_ODS, $this->getName() . ".ods");
$filename .= ".ods";
$excel = new Excel(Excel::FORMAT_ODS, $filename);
break;
case self::FORMAT_CSV:
default:
$excel = new Excel(Excel::FORMAT_CSV, $this->getName() . ".csv");
$filename .= ".csv";
$excel = new Excel(Excel::FORMAT_CSV, $filename);
break;
}
// override the open to browser by the writer
if (!empty($absoluteDirectoryPath)) {
if (!is_dir($absoluteDirectoryPath)) {
@mkdir($absoluteDirectoryPath, 0777, true);
}
$filePath = \p4string::addEndSlash($absoluteDirectoryPath) . $filename;
@touch($filePath);
$excel->getWriter()->openToFile($filePath);
}
$excel->addRow($this->getColumnTitles());
$n = 0;
@@ -154,4 +170,8 @@ abstract class Report
$excel->render();
}
private function normalizeString($filename)
{
return (new Slugify())->slugify($filename, '-');
}
}

View File

@@ -0,0 +1,121 @@
<?php
/**
* This file is part of Phraseanet
*
* (c) 2005-2016 Alchemy
*
* For the full copyright and license information, please view the LICENSE
* file that was distributed with this source code.
*/
namespace Alchemy\Phrasea\Report;
use Alchemy\Phrasea\Exception\InvalidArgumentException;
class ReportCountAssets extends Report
{
/* those vars will be set once by computeVars() */
private $name = null;
private $sql = null;
private $columnTitles = [];
private $keyName = null;
public function getName()
{
$this->computeVars();
return $this->name;
}
public function getColumnTitles()
{
$this->computeVars();
return $this->columnTitles;
}
public function getKeyName()
{
$this->computeVars();
return $this->keyName;
}
public function getAllRows($callback)
{
$this->computeVars();
$stmt = $this->databox->get_connection()->executeQuery($this->sql, []);
while (($row = $stmt->fetch())) {
$callback($row);
}
$stmt->closeCursor();
}
private function computeVars()
{
if(!is_null($this->name)) {
// vars already computed
return;
}
$sqlWhereDate = " date>=" . $this->databox->get_connection()->quote($this->parms['dmin']);
if ($this->parms['dmax']) {
$sqlWhereDate .= " AND date<=" . $this->databox->get_connection()->quote($this->parms['dmax']);
}
switch($this->parms['group']) {
case 'added,year':
$this->name = "Number of added assets per year";
$this->columnTitles = ['year', 'nb'];
$this->sql = "SELECT YEAR(date) AS year,COUNT(record_id) AS nb \n"
. "FROM log_docs WHERE action='add' AND " . $sqlWhereDate ."\n"
. "GROUP BY year";
break;
case 'added,year,month':
$this->name = "Number of added assets per year, month";
$this->columnTitles = ['year', 'month', 'nb'];
$this->sql = "SELECT YEAR(date) AS year, MONTH(date) AS month, COUNT(record_id) AS nb \n"
. "FROM log_docs WHERE action='add' AND " . $sqlWhereDate ."\n"
. "GROUP BY year, month";
break;
case 'downloaded,year':
$this->name = "Number of downloaded assets per year";
$this->columnTitles = ['year', 'nb'];
$this->sql = "SELECT YEAR(date) AS year, COUNT(record_id) AS nb \n"
. "FROM log_docs WHERE (action='download' OR action='mail') AND " . $sqlWhereDate ."\n"
. "GROUP BY year";
break;
case 'downloaded,year,month':
$this->name = "Number of downloaded assets per year, month";
$this->columnTitles = ['year', 'month', 'nb'];
$this->sql = "SELECT YEAR(date) AS year, MONTH(date) AS month, COUNT(record_id) AS nb \n"
. "FROM log_docs WHERE (action='download' OR action='mail') AND " . $sqlWhereDate ."\n"
. "GROUP BY year, month";
break;
case 'downloaded,year,month,action':
$this->name = "Number of downloaded assets per year, month, action";
$this->columnTitles = ['action', 'year', 'month', 'nb'];
$this->sql = "SELECT action, YEAR(date) AS year, MONTH(date) AS month, COUNT(record_id) AS nb \n"
. "FROM log_docs WHERE (action='download' OR action='mail') AND " . $sqlWhereDate ."\n"
. "GROUP BY year, month, action";
break;
case 'most-downloaded':
$this->name = "Most downloaded assets";
$this->columnTitles = ['record_id', 'nb'];
$this->sql = "SELECT record_id, COUNT(record_id) AS nb \n"
. "FROM log_docs WHERE (action='download' OR action='mail') AND " . $sqlWhereDate ."\n"
. "GROUP BY record_id ORDER BY nb DESC limit 20";
break;
default:
throw new InvalidArgumentException('invalid "group" argument');
break;
}
}
}

View File

@@ -80,16 +80,16 @@ class ReportDownloads extends Report
switch ($this->parms['group']) {
case null:
$this->name = "Downloads";
$this->columnTitles = ['id', 'usrid', 'user', 'fonction', 'societe', 'activite', 'pays', 'date', 'record_id', 'coll_id', 'subdef'];
$this->columnTitles = ['id', 'usrid', 'user', 'fonction', 'societe', 'activite', 'pays', 'date', 'record_id', 'coll_id', 'subdef', 'action'];
if($this->parms['anonymize']) {
$sql = "SELECT `ld`.`id`, `l`.`usrid`, '-' AS `user`, '-' AS `fonction`, '-' AS `societe`, '-' AS `activite`, '-' AS `pays`,\n"
. " `ld`.`date`, `ld`.`record_id`, `ld`.`coll_id`, `ld`.`final`"
. " `ld`.`date`, `ld`.`record_id`, `ld`.`coll_id`, `ld`.`final`, `ld`.`action`"
. " FROM `log_docs` AS `ld` INNER JOIN `log` AS `l` ON `l`.`id`=`ld`.`log_id`\n"
. " WHERE {{GlobalFilter}}";
}
else {
$sql = "SELECT `ld`.`id`, `l`.`usrid`, `l`.`user`, `l`.`fonction`, `l`.`societe`, `l`.`activite`, `l`.`pays`,\n"
. " `ld`.`date`, `ld`.`record_id`, `ld`.`coll_id`, `ld`.`final`"
. " `ld`.`date`, `ld`.`record_id`, `ld`.`coll_id`, `ld`.`final`, `ld`.`action`"
. " FROM `log_docs` AS `ld` INNER JOIN `log` AS `l` ON `l`.`id`=`ld`.`log_id`\n"
. " WHERE {{GlobalFilter}}";
}
@@ -151,7 +151,7 @@ class ReportDownloads extends Report
$subdefsToReport = join(',', $subdefsToReport);
$filter = "`action`='download' AND `ld`.`coll_id` IN(" . join(',', $collIds) . ")\n"
$filter = "(`action`='download' OR `action`='mail') AND `ld`.`coll_id` IN(" . join(',', $collIds) . ")\n"
. " AND `l`.`usrid`>0\n"
. " AND `ld`.`final` IN(" . $subdefsToReport . ")";

View File

@@ -0,0 +1,89 @@
<?php
/**
* This file is part of Phraseanet
*
* (c) 2005-2016 Alchemy
*
* For the full copyright and license information, please view the LICENSE
* file that was distributed with this source code.
*/
namespace Alchemy\Phrasea\Report;
use Alchemy\Phrasea\Exception\InvalidArgumentException;
class ReportUsers extends Report
{
/* those vars will be set once by computeVars() */
private $name = null;
private $sql = null;
private $columnTitles = [];
private $keyName = null;
public function getName()
{
$this->computeVars();
return $this->name;
}
public function getColumnTitles()
{
$this->computeVars();
return $this->columnTitles;
}
public function getKeyName()
{
$this->computeVars();
return $this->keyName;
}
public function getAllRows($callback)
{
$this->computeVars();
// use appbox connection
$stmt = $this->databox->get_appbox()->get_connection()->executeQuery($this->sql, []);
while (($row = $stmt->fetch())) {
$callback($row);
}
$stmt->closeCursor();
}
private function computeVars()
{
if(!is_null($this->name)) {
// vars already computed
return;
}
$sqlWhereDate = " created>=" . $this->databox->get_appbox()->get_connection()->quote($this->parms['dmin']);
if ($this->parms['dmax']) {
$sqlWhereDate .= " AND created<=" . $this->databox->get_appbox()->get_connection()->quote($this->parms['dmax']);
}
switch($this->parms['group']) {
case 'added,year':
$this->name = "Number of newly added user per year";
$this->columnTitles = ['year', 'nb'];
$this->sql = "SELECT YEAR(created) AS year, COUNT(id) AS nb \n "
. "FROM Users \n"
. "WHERE ". $sqlWhereDate . "\n"
. "GROUP BY year";
break;
case 'added,year,month':
$this->name = "Number of newly added user per year,month";
$this->columnTitles = ['year', 'month', 'nb'];
$this->sql = "SELECT YEAR(created) AS year, MONTH(created) AS month, COUNT(id) AS nb \n "
. "FROM Users \n"
. "WHERE ". $sqlWhereDate . "\n"
. "GROUP BY year, month";
break;
default:
throw new InvalidArgumentException('invalid "group" argument');
break;
}
}
}