MISP
/
MISP
mirror of https://github.com/MISP/MISP
2
2
Fork 0
Code Issues Releases Wiki Activity

Merge pull request #9523 from JakubOnderka/browscap-apcu-cache 

Browscap apcu cache
pull/9524/head
Jakub Onderka 2024-01-28 13:28:04 +01:00 committed by GitHub
parent 6bf4c9d7e1 f0a2c9c8e9
commit 91e462098e
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
5 changed files with 199 additions and 35747 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

55
app/Console/Command/UserShell.php
View File

@ -3,10 +3,11 @@
/**
* @property User $User
* @property Log $Log
* @property UserLoginProfile $UserLoginProfile
*/
class UserShell extends AppShell
{
public $uses = ['User', 'Log'];
public $uses = ['User', 'Log', 'UserLoginProfile'];
public function getOptionParser()
{
@ -29,7 +30,7 @@ class UserShell extends AppShell
'help' => __('Get information about given authkey.'),
'parser' => [
'arguments' => [
'authkey' => ['help' => __('Authentication key. If not provide, it will be read from STDIN.')],
'authkey' => ['help' => __('Authentication key. If not provided, it will be read from STDIN.')],
],
]
]);
@ -112,6 +113,14 @@ class UserShell extends AppShell
],
],
]);
$parser->addSubcommand('ip_country', [
'help' => __('Get country for given IP address'),
'parser' => [
'arguments' => [
'ip' => ['help' => __('IPv4 or IPv6 address.'), 'required' => true],
]
],
]);
$parser->addSubcommand('require_password_change_for_old_passwords', [
'help' => __('Trigger forced password change on next login for users with an old (older than x days) password.'),
'parser' => [
@ -188,11 +197,7 @@ class UserShell extends AppShell
public function authkey()
{
if (isset($this->args[0])) {
$authkey = $this->args[0];
} else {
$authkey = fgets(STDIN); // read line from STDIN
}
$authkey = $this->args[0] ?? fgets(STDIN);
$authkey = trim($authkey);
if (strlen($authkey) !== 40) {
$this->error('Authkey has not valid format.');
@ -353,7 +358,7 @@ class UserShell extends AppShell
$conditions = ['User.disabled' => false]; // fetch just not disabled users
$userId = isset($this->args[0]) ? $this->args[0] : null;
$userId = $this->args[0] ?? null;
if ($userId) {
$conditions['OR'] = [
'User.id' => $userId,
@ -412,7 +417,7 @@ class UserShell extends AppShell
}
$user = $this->getUser($userId);
# validate new authentication key if provided
// validate new authentication key if provided
if (!empty($newkey) && (strlen($newkey) != 40 || !ctype_alnum($newkey))) {
$this->error('The new auth key needs to be 40 characters long and only alphanumeric.');
}
@ -447,7 +452,7 @@ class UserShell extends AppShell
$this->out('<warning>Storing user IP addresses is disabled.</warning>');
}
$ips = $this->User->setupRedisWithException()->smembers('misp:user_ip:' . $user['id']);
$ips = RedisTool::init()->smembers('misp:user_ip:' . $user['id']);
if ($this->params['json']) {
$this->out($this->json($ips));
@ -470,36 +475,50 @@ class UserShell extends AppShell
$this->out('<warning>Storing user IP addresses is disabled.</warning>');
}
$userId = $this->User->setupRedisWithException()->get('misp:ip_user:' . $ip);
$userId = RedisTool::init()->get('misp:ip_user:' . $ip);
if (empty($userId)) {
$this->out('No hits.');
$this->_stop();
}
$user = $this->User->find('first', array(
$user = $this->User->find('first', [
'recursive' => -1,
'conditions' => array('User.id' => $userId),
'conditions' => ['User.id' => $userId],
'fields' => ['id', 'email'],
));
]);
if (empty($user)) {
$this->error("User with ID $userId doesn't exists anymore.");
}
$ipCountry = $this->UserLoginProfile->countryByIp($ip);
if ($this->params['json']) {
$this->out($this->json([
'ip' => $ip,
'id' => $user['User']['id'],
'email' => $user['User']['email'],
'country' => $ipCountry,
]));
} else {
$this->out(sprintf(
'%s==============================%sIP: %s%s==============================%sUser #%s: %s%s==============================%s',
PHP_EOL, PHP_EOL, $ip, PHP_EOL, PHP_EOL, $user['User']['id'], $user['User']['email'], PHP_EOL, PHP_EOL
));
$this->hr();
$this->out("IP: $ip (country $ipCountry)");
$this->hr();
$this->out("User #{$user['User']['id']}: {$user['User']['email']}");
$this->hr();
}
}
public function ip_country()
{
list($ip) = $this->args;
if (!filter_var($ip, FILTER_VALIDATE_IP)) {
$this->error("IP `$ip` is not valid IPv4 or IPv6 address");
}
$this->out($this->UserLoginProfile->countryByIp($ip));
}
public function require_password_change_for_old_passwords()
{
list($days) = $this->args;

153
app/Lib/Tools/ApcuCacheTool.php
View File

@ -14,9 +14,19 @@ class ApcuCacheTool implements \Psr\SimpleCache\CacheInterface
$this->prefix = $prefix;
}
/**
* Fetches a value from the cache.
*
* @param string $key The unique key of this item in the cache.
* @param mixed $default Default value to return if the key does not exist.
*
* @return mixed The value of the item from the cache, or $default in case of cache miss.
*
* @throws \Psr\SimpleCache\InvalidArgumentException
* MUST be thrown if the $key string is not a legal value.
*/
public function get($key, $default = null)
{
$success = false;
$value = \apcu_fetch("$this->prefix:$key", $success);
if ($success) {
return $value;
@ -24,48 +34,165 @@ class ApcuCacheTool implements \Psr\SimpleCache\CacheInterface
return $default;
}
/**
* Persists data in the cache, uniquely referenced by a key with an optional expiration TTL time.
*
* @param string $key The key of the item to store.
* @param mixed $value The value of the item to store, must be serializable.
* @param null|int|\DateInterval $ttl Optional. The TTL value of this item. If no value is sent and
* the driver supports TTL then the library may set a default value
* for it or let the driver take care of that.
*
* @return bool True on success and false on failure.
*
* @throws \Psr\SimpleCache\InvalidArgumentException
* MUST be thrown if the $key string is not a legal value.
*/
public function set($key, $value, $ttl = null)
{
return \apcu_store("$this->prefix:$key", $value, $ttl === null ? 0 : $ttl);
return \apcu_store("$this->prefix:$key", $value, $this->tllToInt($ttl));
}
/**
* Delete an item from the cache by its unique key.
*
* @param string $key The unique cache key of the item to delete.
*
* @return bool True if the item was successfully removed. False if there was an error.
*
* @throws \Psr\SimpleCache\InvalidArgumentException
* MUST be thrown if the $key string is not a legal value.
*/
public function delete($key)
{
return \apcu_delete("$this->prefix:$key");
}
/**
* Wipes clean the entire cache's keys.
*
* @return bool True on success and false on failure.
*/
public function clear()
{
foreach (new APCUIterator("/^$this->prefix:/") as $item) {
\apcu_delete($item['key']);
}
$iterator = new APCUIterator(
'/^' . preg_quote($this->prefix . ':', '/') . '/',
APC_ITER_NONE
);
return \apcu_delete($iterator);
}
/**
* Obtains multiple cache items by their unique keys.
*
* @param iterable $keys A list of keys that can obtained in a single operation.
* @param mixed $default Default value to return for keys that do not exist.
*
* @return iterable A list of key => value pairs. Cache keys that do not exist or are stale will have $default as value.
*
* @throws \Psr\SimpleCache\InvalidArgumentException
* MUST be thrown if $keys is neither an array nor a Traversable,
* or if any of the $keys are not a legal value.
*/
public function getMultiple($keys, $default = null)
{
foreach ($keys as $key) {
yield $key => $this->get($key, $default);
$keysToFetch = $this->keysToFetch($keys);
$values = \apcu_fetch($keysToFetch);
foreach ($keysToFetch as $keyToFetch) {
if (!isset($values[$keyToFetch])) {
$values[$keyToFetch] = $default;
}
}
return $values;
}
/**
* Persists a set of key => value pairs in the cache, with an optional TTL.
*
* @param iterable $values A list of key => value pairs for a multiple-set operation.
* @param null|int|\DateInterval $ttl Optional. The TTL value of this item. If no value is sent and
* the driver supports TTL then the library may set a default value
* for it or let the driver take care of that.
*
* @return bool True on success and false on failure.
*
* @throws \Psr\SimpleCache\InvalidArgumentException
* MUST be thrown if $values is neither an array nor a Traversable,
* or if any of the $values are not a legal value.
*/
public function setMultiple($values, $ttl = null)
{
$dataToSave = [];
foreach ($values as $key => $value) {
$this->set($key, $value, $ttl);
$dataToSave["$this->prefix:$key"] = $value;
}
return true;
return \apcu_store($dataToSave, null, $this->tllToInt($ttl));
}
/**
* Deletes multiple cache items in a single operation.
*
* @param iterable $keys A list of string-based keys to be deleted.
*
* @return bool True if the items were successfully removed. False if there was an error.
*
* @throws \Psr\SimpleCache\InvalidArgumentException
* MUST be thrown if $keys is neither an array nor a Traversable,
* or if any of the $keys are not a legal value.
*/
public function deleteMultiple($keys)
{
foreach ($keys as $key) {
$this->delete($key);
}
return true;
$keysToDelete = $this->keysToFetch($keys);
return \apcu_delete($keysToDelete);
}
/**
* Determines whether an item is present in the cache.
*
* NOTE: It is recommended that has() is only to be used for cache warming type purposes
* and not to be used within your live applications operations for get/set, as this method
* is subject to a race condition where your has() will return true and immediately after,
* another script can remove it making the state of your app out of date.
*
* @param string $key The cache item key.
*
* @return bool
*
* @throws \Psr\SimpleCache\InvalidArgumentException
* MUST be thrown if the $key string is not a legal value.
*/
public function has($key)
{
return \apcu_exists("$this->prefix:$key");
}
/**
* @param iterable $keys
* @return array
*/
private function keysToFetch(iterable $keys): array
{
$keysToFetch = [];
foreach ($keys as $key) {
$keysToFetch[] = "$this->prefix:$key";
}
return $keysToFetch;
}
/**
* @param null|int|\DateInterval $ttl
* @return int
*/
private function tllToInt($ttl = null): int
{
if ($ttl === null) {
return 0;
} elseif (is_int($ttl)) {
return $ttl;
} elseif ($ttl instanceof \DateInterval) {
return $ttl->days * 86400 + $ttl->h * 3600 + $ttl->i * 60 + $ttl->s;
} else {
throw new \Psr\SimpleCache\InvalidArgumentException("Invalid ttl value '$ttl' provided.");
}
}
}

36
app/Model/UserLoginProfile.php
View File

@ -36,7 +36,7 @@ class UserLoginProfile extends AppModel
];
const BROWSER_CACHE_DIR = APP . DS . 'tmp' . DS . 'browscap';
const BROWSER_INI_FILE = APP . DS . 'files' . DS . 'browscap'. DS . 'browscap.ini'; // Browscap file managed by MISP - https://browscap.org/stream?q=Lite_PHP_BrowsCapINI
const BROWSER_INI_FILE = APP . DS . 'files' . DS . 'browscap'. DS . 'browscap.ini.gz'; // Browscap file managed by MISP - https://browscap.org/stream?q=Lite_PHP_BrowsCapINI
const GEOIP_DB_FILE = APP . DS . 'files' . DS . 'geo-open' . DS . 'GeoOpen-Country.mmdb'; // GeoIP file managed by MISP - https://data.public.lu/en/datasets/geo-open-ip-address-geolocation-per-country-in-mmdb-format/
private $userProfile;
@ -61,13 +61,32 @@ class UserLoginProfile extends AppModel
} catch (\BrowscapPHP\Exception $e) {
$this->log("Browscap - building new cache from browscap.ini file.", LOG_INFO);
$bcUpdater = new \BrowscapPHP\BrowscapUpdater($cache, $logger);
$bcUpdater->convertFile(UserLoginProfile::BROWSER_INI_FILE);
$bcUpdater->convertString(FileAccessTool::readCompressedFile(UserLoginProfile::BROWSER_INI_FILE));
}
$bc = new \BrowscapPHP\Browscap($cache, $logger);
return $bc->getBrowser();
}
/**
* @param string $ip
* @return string|null
*/
public function countryByIp($ip)
{
if (class_exists('GeoIp2\Database\Reader')) {
$geoDbReader = new GeoIp2\Database\Reader(UserLoginProfile::GEOIP_DB_FILE);
try {
$record = $geoDbReader->country($ip);
return $record->country->isoCode;
} catch (InvalidArgumentException $e) {
$this->logException("Could not get country code for IP address", $e, LOG_NOTICE);
return null;
}
}
return null;
}
public function beforeSave($options = [])
{
$this->data['UserLoginProfile']['hash'] = $this->hash($this->data['UserLoginProfile']);
@ -105,18 +124,7 @@ class UserLoginProfile extends AppModel
$browser->browser = "browser";
}
$ip = $this->_remoteIp();
if (class_exists('GeoIp2\Database\Reader')) {
try {
$geoDbReader = new GeoIp2\Database\Reader(UserLoginProfile::GEOIP_DB_FILE);
$record = $geoDbReader->country($ip);
$country = $record->country->isoCode;
} catch (InvalidArgumentException $e) {
$this->logException("Could not get country code for IP address", $e);
$country = 'None';
}
} else {
$country = 'None';
}
$country = $this->countryByIp($ip) ?? 'None';
$this->userProfile = [
'user_agent' => $_SERVER['HTTP_USER_AGENT'] ?? null,
'ip' => $ip,

35702
app/files/browscap/browscap.ini
View File

File diff suppressed because it is too large Load Diff

BIN
app/files/browscap/browscap.ini.gz Normal file
View File

Binary file not shown.