<?php

#   Copyright (C) 2006-2012 Tobias Leupold <tobias.leupold@web.de>
#
#   b8 - A statistical ("Bayesian") spam filter written in PHP 5
#
#   This program is free software; you can redistribute it and/or modify it
#   under the terms of the GNU Lesser General Public License as published by
#   the Free Software Foundation in version 2.1 of the License.
#
#   This program is distributed in the hope that it will be useful, but
#   WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
#   or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public
#   License for more details.
#
#   You should have received a copy of the GNU Lesser General Public License
#   along with this program; if not, write to the Free Software Foundation,
#   Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.

/**
 * Copyright (C) 2006-2012 Tobias Leupold <tobias.leupold@web.de>
 *
 * @license LGPL 2.1
 * @access public
 * @package b8
 * @author Tobias Leupold
 * @author Oliver Lillie (original PHP 5 port)
 */

class b8
{

	const DBVERSION = 3;

	public $config = array(
		'lexer'        => 'default',
		'degenerator'  => 'default',
		'storage'      => 'mysql',
		'use_relevant' => 15,
		'min_dev'      => 0.2,
		'rob_s'        => 0.3,
		'rob_x'        => 0.5
	);
	
	public $storage     = NULL;
	public $lexer       = NULL;
	public $degenerator = NULL;
	
	private $_token_data = NULL;
	
	const SPAM    = 'spam';
	const HAM     = 'ham';
	const LEARN   = 'learn';
	const UNLEARN = 'unlearn';
	
	const TRAINER_CATEGORY_FAIL = 'TRAINER_CATEGORY_FAIL';

	/**
	 * Constructs b8
	 *
	 * @access public
	 * @return void
	 */
	
	function __construct(
		$config             = array(),
		$config_storage     = array(),
		$config_lexer       = array(),
		$config_degenerator = array()
	)
	{
	
		# Validate config data
		
		foreach ($config as $name => $value) {
		
			switch($name) {
			
				case 'min_dev':
				case 'rob_s':
				case 'rob_x':
					$this->config[$name] = (float) $value;
					break;
					
				case 'use_relevant':
					$this->config[$name] = (int) $value;
					break;
					
				case 'lexer':
				case 'degenerator':
				case 'storage':
					$this->config[$name] = (string) $value;
					break;
				
				default:
					throw new Exception("b8: Unknown configuration key: \"$name\"");
			}
			
		}
		
		# Setup the degenerator class
		
		$class = $this->_load_class('degenerator', $this->config['degenerator']);
		
		if($class === FALSE)
			throw new Exception("b8: Could not load class definition file for degenerator \"{$this->config['degenerator']}\"");
		
		$this->degenerator = new $class($config_degenerator);
		
		# Setup the lexer class
		
		$class = $this->_load_class('lexer', $this->config['lexer']);
		
		if($class === FALSE)
			throw new Exception("b8: Could not load class definition file for lexer \"{$this->config['lexer']}\"");
		
		$this->lexer = new $class($config_lexer);
		
		# Setup the storage backend
		
		$class = $this->_load_class('storage', 'base');
		
		if($class === FALSE)
			throw new Exception("b8: Could not load class definition file for the storage base class");
		
		$class = $this->_load_class('storage', $this->config['storage']);
		
		if($class === FALSE)
			throw new Exception("b8: Could not load class definition file for storage backend \"{$this->config['storage']}\"");
		
		$this->storage = new $class($config_storage, $this->degenerator);
		
	}
	
	/**
	 * Load a class file if a class has not been defined yet.
	 *
	 * @access private
	 * @return boolean Returns TRUE if everything is okay, otherwise FALSE.
	 */
	
	private function _load_class($class_type, $class_name)
	{
	
		$complete_class_name = "b8_{$class_type}_{$class_name}";
		$class_file = dirname(__FILE__) . DIRECTORY_SEPARATOR . $class_type . DIRECTORY_SEPARATOR . "{$class_type}_{$class_name}.php";
	
		if(class_exists($complete_class_name, FALSE) === FALSE) {
		
			# Check if the requested file actually exists
			if(is_file($class_file) !== TRUE or is_readable($class_file) !== TRUE)
				return FALSE;
			
			# Include it
			$included = require_once($class_file);
			
			if($included === FALSE or class_exists($complete_class_name, FALSE) === FALSE)
				return FALSE;
				
		}
		
		return $complete_class_name;
		
	}
	
	/**
	 * Classifies a text
	 *
	 * @access public
	 * @param string $text
	 * @return float The rating between 0 (ham) and 1 (spam)
	 */
	
	public function classify($text)
	{
	
		# Get the internal database variables, containing the number of ham and
		# spam texts so the spam probability can be calculated in relation to them
		$internals = $this->storage->get_internals();
		
		# Calculate the spamminess of all tokens
		
		# Get all tokens we want to rate
		$tokens = $this->lexer->get_tokens($text);
		
		# Check if the lexer failed
		# (if so, $tokens will be a lexer error code, if not, $tokens will be an array)
		if(!is_array($tokens))
			return $tokens;
			
		# Fetch all available data for the token set from the database
		$this->_token_data = $this->storage->get(array_keys($tokens));
		
		# Calculate the spamminess and importance for each token (or a degenerated form of it)
		
		$word_count = array();
		$rating     = array();
		$importance = array();
		
		foreach($tokens as $word => $count) {
		
			$word_count[$word] = $count;
			
			# Although we only call this function only here ... let's do the
			# calculation stuff in a function to make this a bit less confusing ;-)
			$rating[$word] = $this->_get_probability($word, $internals['texts_ham'], $internals['texts_spam']);
			
			$importance[$word] = abs(0.5 - $rating[$word]);
			
		}
		
		# Order by importance
		arsort($importance);
		reset($importance);
		
		# Get the most interesting tokens (use all if we have less than the given number)
		
		$relevant = array();
		
		for($i = 0; $i < $this->config['use_relevant']; $i++) {
		
			if($tmp = each($importance)) {
				
				# Important tokens remain
				
				# If the token's rating is relevant enough, use it
				
				if(abs(0.5 - $rating[$tmp['key']]) > $this->config['min_dev']) {
				
					# Tokens that appear more than once also count more than once
					
					for($x = 0, $l = $word_count[$tmp['key']]; $x < $l; $x++)
						array_push($relevant, $rating[$tmp['key']]);
						
				}
				
			}
			
			else {
				# We have less words as we want to use, so we
				# already use what we have and can break here
				break;
			}
			
		}
		
		# Calculate the spamminess of the text (thanks to Mr. Robinson ;-)
		# We set both hamminess and spamminess to 1 for the first multiplying
		$hamminess  = 1;
		$spamminess = 1;
		
		# Consider all relevant ratings
		foreach($relevant as $value) {
			$hamminess  *= (1.0 - $value);
			$spamminess *= $value;
		}
		
		# If no token was good for calculation, we really don't know how
		# to rate this text; so we assume a spam and ham probability of 0.5
		
		if($hamminess === 1 and $spamminess === 1) {
			$hamminess = 0.5;
			$spamminess = 0.5;
			$n = 1;
		}
		else {
			# Get the number of relevant ratings
			$n = count($relevant);
		}
		
		# Calculate the combined rating
		
		# The actual hamminess and spamminess
		$hamminess  = 1 - pow($hamminess,  (1 / $n));
		$spamminess = 1 - pow($spamminess, (1 / $n));
		
		# Calculate the combined indicator
		$probability = ($hamminess - $spamminess) / ($hamminess + $spamminess);
		
		# We want a value between 0 and 1, not between -1 and +1, so ...
		$probability = (1 + $probability) / 2;
		
		# Alea iacta est
		return $probability;
		
	}
	
	/**
	 * Calculate the spamminess of a single token also considering "degenerated" versions
	 *
	 * @access private
	 * @param string $word
	 * @param string $texts_ham
	 * @param string $texts_spam
	 * @return void
	 */
	
	private function _get_probability($word, $texts_ham, $texts_spam)
	{
	
		# Let's see what we have!
		
		if(isset($this->_token_data['tokens'][$word]) === TRUE) {
			# The token is in the database, so we can use it's data as-is
			# and calculate the spamminess of this token directly
			return $this->_calc_probability($this->_token_data['tokens'][$word], $texts_ham, $texts_spam);
		}
		
		# The token was not found, so do we at least have similar words?
		
		if(isset($this->_token_data['degenerates'][$word]) === TRUE) {
		
			# We found similar words, so calculate the spamminess for each one
			# and choose the most important one for the further calculation
			
			# The default rating is 0.5 simply saying nothing
			$rating = 0.5;
			
			foreach($this->_token_data['degenerates'][$word] as $degenerate => $count) {
			
				# Calculate the rating of the current degenerated token
				$rating_tmp = $this->_calc_probability($count, $texts_ham, $texts_spam);
				
				# Is it more important than the rating of another degenerated version?
				if(abs(0.5 - $rating_tmp) > abs(0.5 - $rating))
					$rating = $rating_tmp;
				
			}
			
			return $rating;
			
		}
		
		else {
			# The token is really unknown, so choose the default rating
			# for completely unknown tokens. This strips down to the
			# robX parameter so we can cheap out the freaky math ;-)
			return $this->config['rob_x'];
		}
		
	}
	
	/**
	 * Do the actual spamminess calculation of a single token
	 *
	 * @access private
	 * @param array $data
	 * @param string $texts_ham
	 * @param string $texts_spam
	 * @return void
	 */
	
	private function _calc_probability($data, $texts_ham, $texts_spam)
	{
	
		# Calculate the basic probability as proposed by Mr. Graham
		
		# But: consider the number of ham and spam texts saved instead of the
		# number of entries where the token appeared to calculate a relative
		# spamminess because we count tokens appearing multiple times not just
		# once but as often as they appear in the learned texts.
		
		$rel_ham = $data['count_ham'];
		$rel_spam = $data['count_spam'];
		
		if($texts_ham > 0)
			$rel_ham = $data['count_ham'] / $texts_ham;
			
		if($texts_spam > 0)
			$rel_spam = $data['count_spam'] / $texts_spam;
			
		$rating = $rel_spam / ($rel_ham + $rel_spam);
		
		# Calculate the better probability proposed by Mr. Robinson
		$all = $data['count_ham'] + $data['count_spam'];
		return (($this->config['rob_s'] * $this->config['rob_x']) + ($all * $rating)) / ($this->config['rob_s'] + $all);
		
	}
	
	/**
	 * Check the validity of the category of a request
	 *
	 * @access private
	 * @param string $category
	 * @return void
	 */
	
	private function _check_category($category)
	{
		return $category === self::HAM or $category === self::SPAM;
	}
	
	/**
	 * Learn a reference text
	 *
	 * @access public
	 * @param string $text
	 * @param const $category Either b8::SPAM or b8::HAM
	 * @return void
	 */
	
	public function learn($text, $category)
	{
		return $this->_process_text($text, $category, self::LEARN);
	}
	
	/**
	 * Unlearn a reference text
	 *
	 * @access public
	 * @param string $text
	 * @param const $category Either b8::SPAM or b8::HAM
	 * @return void
	 */
	
	public function unlearn($text, $category)
	{
		return $this->_process_text($text, $category, self::UNLEARN);
	}
	
	/**
	 * Does the actual interaction with the storage backend for learning or unlearning texts
	 *
	 * @access private
	 * @param string $text
	 * @param const $category Either b8::SPAM or b8::HAM
	 * @param const $action Either b8::LEARN or b8::UNLEARN
	 * @return void
	 */
	
	private function _process_text($text, $category, $action)
	{
	
		# Look if the request is okay
		if($this->_check_category($category) === FALSE)
			return self::TRAINER_CATEGORY_FAIL;
		
		# Get all tokens from $text
		$tokens = $this->lexer->get_tokens($text);
		
		# Check if the lexer failed
		# (if so, $tokens will be a lexer error code, if not, $tokens will be an array)
		if(!is_array($tokens))
			return $tokens;
		
		# Pass the tokens and what to do with it to the storage backend
		return $this->storage->process_text($tokens, $category, $action);
		
	}
	
}

?>