php/yii2
1
0
Fork 0
mirror of https://github.com/yiisoft/yii2.git synced 2025-11-21 00:54:53 +08:00
Code Issues Packages Projects Releases Wiki Activity

basic CRUD for elastic search WIP

Browse Source
This commit is contained in:
Carsten Brandt
2013-09-30 18:47:31 +02:00
parent a94886fafd
commit 955bf7daaf
6 changed files with 663 additions and 278 deletions
Download Patch File Download Diff File Expand all files Collapse all files

109
framework/yii/elasticsearch/ActiveQuery.php
View File

@@ -6,6 +6,8 @@
*/
namespace yii\elasticsearch;
use Guzzle\Http\Client;
use Guzzle\Http\Exception\MultiTransferException;
use yii\base\NotSupportedException;
use yii\db\Exception;
use yii\helpers\Json;
@@ -77,15 +79,21 @@ class ActiveQuery extends \yii\base\Component
* of [[modelClass]] will be created to represent each record.
*/
public $asArray;
/**
* @var array the columns being selected. For example, `array('id', 'name')`.
* This is used to construct the SELECT clause in a SQL statement. If not set, if means selecting all columns.
* @see select()
*/
public $select;
/**
* @var array the query condition.
* @see where()
*/
public $where;
/**
* @var integer maximum number of records to be returned. If not set or less than 0, it means no limit.
* @var integer maximum number of records to be returned. If not set or less than 0, it means no limit. TODO infinite possible in ES?
*/
public $limit;
public $limit = 10;
/**
* @var integer zero-based offset from where the records are to be returned.
* If not set, it means starting from the beginning.
@@ -128,12 +136,10 @@ class ActiveQuery extends \yii\base\Component
// TODO add support for orderBy
$data = $this->executeScript('All');
$rows = array();
print_r($data);
foreach($data as $dataRow) {
$row = array();
$c = count($dataRow);
for($i = 0; $i < $c; ) {
$row[$dataRow[$i++]] = $dataRow[$i++];
}
$row = $dataRow['_source'];
$row['id'] = $dataRow['_id'];
$rows[] = $row;
}
if (!empty($rows)) {
@@ -157,14 +163,11 @@ class ActiveQuery extends \yii\base\Component
{
// TODO add support for orderBy
$data = $this->executeScript('One');
if ($data === array()) {
if (!isset($data['_source'])) {
return null;
}
$row = array();
$c = count($data);
for($i = 0; $i < $c; ) {
$row[$data[$i++]] = $data[$i++];
}
$row = $data['_source'];
$row['id'] = $data['_id'];
if ($this->asArray) {
$model = $row;
} else {
@@ -284,12 +287,13 @@ class ActiveQuery extends \yii\base\Component
{
if (($data = $this->findByPk($type)) === false) {
$modelClass = $this->modelClass;
/** @var Connection $db */
$db = $modelClass::getDb();
$http = $modelClass::getDb()->http();
$method = 'build' . $type;
$script = $db->getLuaScriptBuilder()->$method($this, $columnName);
return $db->executeCommand('EVAL', array($script, 0));
$url = '/' . $modelClass::indexName() . '/' . $modelClass::indexType() . '/_search';
$query = $modelClass::getDb()->getQueryBuilder()->build($this);
$response = $http->post($url, null, Json::encode($query))->send();
$data = Json::decode($response->getBody(true));
return $data['hits']['hits'];
}
return $data;
}
@@ -301,46 +305,47 @@ class ActiveQuery extends \yii\base\Component
{
$modelClass = $this->modelClass;
if (is_array($this->where) && !isset($this->where[0]) && $modelClass::isPrimaryKey(array_keys($this->where))) {
/** @var Connection $db */
$db = $modelClass::getDb();
/** @var Client $http */
$http = $modelClass::getDb()->http();
$pks = (array) reset($this->where);
$start = $this->offset === null ? 0 : $this->offset;
$i = 0;
$data = array();
$url = '/' . $modelClass::indexName() . '/' . $modelClass::indexType() . '/';
$query = array('docs' => array());
foreach($pks as $pk) {
if (++$i > $start && ($this->limit === null || $i <= $start + $this->limit)) {
$request = $db->http()->get($url . $pk);
$response = $request->send();
if ($response->getStatusCode() == 404) {
// ignore?
} else {
$data[] = Json::decode($response->getBody(true));
if ($type === 'One' && $this->orderBy === null) {
break;
}
}
$doc = array('_id' => $pk);
if (!empty($this->select)) {
$doc['fields'] = $this->select;
}
$query['docs'][] = $doc;
}
$url = '/' . $modelClass::indexName() . '/' . $modelClass::indexType() . '/_mget';
$response = $http->post($url, null, Json::encode($query))->send();
$data = Json::decode($response->getBody(true));
$start = $this->offset === null ? 0 : $this->offset;
$data = array_slice($data['docs'], $start, $this->limit);
// TODO support orderBy
switch($type) {
case 'All':
return $data;
case 'One':
return reset($data);
return empty($data) ? null : reset($data);
case 'Column':
// TODO support indexBy
$column = array();
foreach($data as $dataRow) {
$row = array();
$c = count($dataRow);
for($i = 0; $i < $c; ) {
$row[$dataRow[$i++]] = $dataRow[$i++];
foreach($data as $row) {
$row['_source']['id'] = $row['_id'];
if ($this->indexBy === null) {
$column[] = $row['_source'][$columnName];
} else {
if (is_string($this->indexBy)) {
$key = $row['_source'][$this->indexBy];
} else {
$key = call_user_func($this->indexBy, $row['_source']);
}
$models[$key] = $row;
}
$column[] = $row[$columnName];
}
return $column;
case 'Count':
@@ -413,6 +418,24 @@ class ActiveQuery extends \yii\base\Component
return $this;
}
/**
* Sets the SELECT part of the query.
* @param string|array $columns the columns to be selected.
* Columns can be specified in either a string (e.g. "id, name") or an array (e.g. array('id', 'name')).
* Columns can contain table prefixes (e.g. "tbl_user.id") and/or column aliases (e.g. "tbl_user.id AS user_id").
* The method will automatically quote the column names unless a column contains some parenthesis
* (which means the column contains a DB expression).
* @return Query the query object itself
*/
public function select($columns)
{
if (!is_array($columns)) {
$columns = preg_split('/\s*,\s*/', trim($columns), -1, PREG_SPLIT_NO_EMPTY);
}
$this->select = $columns;
return $this;
}
/**
* Sets the ORDER BY part of the query.
* @param string|array $columns the columns (and the directions) to be ordered by.

123
framework/yii/elasticsearch/ActiveRecord.php
View File

@@ -64,38 +64,42 @@ abstract class ActiveRecord extends \yii\db\ActiveRecord
*/
public static function updateAll($attributes, $condition = null, $params = array())
{
// TODO add support for further options as described in http://www.elasticsearch.org/guide/reference/api/bulk/
if (empty($attributes)) {
return 0;
}
$db = static::getDb();
if (count($condition) != 1 || !isset($condition[reset(static::primaryKey())])) {
throw new NotSupportedException('UpdateAll is only supported by primary key in elasticsearch.');
}
if (isset($attributes[reset(static::primaryKey())])) {
throw new NotSupportedException('Updating the primary key is currently not supported by elasticsearch.');
}
$query = '';
foreach((array) reset($condition) as $pk) {
if (is_array($pk)) {
$pk = reset($pk);
}
$action = Json::encode(array(
"update" => array(
"_id" => $pk,
"_type" => static::indexType(),
"_index" => static::indexName(),
),
));
$data = Json::encode(array(
"doc" => $attributes
));
$query .= $action . "\n" . $data . "\n";
// TODO implement pk change
}
$url = '/' . static::indexName() . '/' . static::indexType() . '/_bulk';
$response = static::getDb()->http()->post($url, array(), $query)->send();
$body = Json::decode($response->getBody(true));
$n=0;
foreach(static::fetchPks($condition) as $pk) {
$newPk = $pk;
$pk = static::buildKey($pk);
$key = static::tableName() . ':a:' . $pk;
// save attributes
$args = array($key);
foreach($attributes as $attribute => $value) {
if (isset($newPk[$attribute])) {
$newPk[$attribute] = $value;
}
$args[] = $attribute;
$args[] = $value;
foreach($body['items'] as $item) {
if ($item['update']['ok']) {
$n++;
}
$newPk = static::buildKey($newPk);
$newKey = static::tableName() . ':a:' . $newPk;
// rename index if pk changed
if ($newPk != $pk) {
$db->executeCommand('MULTI');
$db->executeCommand('HMSET', $args);
$db->executeCommand('LINSERT', array(static::tableName(), 'AFTER', $pk, $newPk));
$db->executeCommand('LREM', array(static::tableName(), 0, $pk));
$db->executeCommand('RENAME', array($key, $newKey));
$db->executeCommand('EXEC');
} else {
$db->executeCommand('HMSET', $args);
}
$n++;
}
return $n;
}
@@ -117,19 +121,7 @@ abstract class ActiveRecord extends \yii\db\ActiveRecord
*/
public static function updateAllCounters($counters, $condition = null, $params = array())
{
if (empty($counters)) {
return 0;
}
$db = static::getDb();
$n=0;
foreach(static::fetchPks($condition) as $pk) {
$key = static::tableName() . ':a:' . static::buildKey($pk);
foreach($counters as $attribute => $value) {
$db->executeCommand('HINCRBY', array($key, $attribute, $value));
}
$n++;
}
return $n;
throw new NotSupportedException('Update Counters is not supported by elasticsearch.');
}
/**
@@ -149,23 +141,36 @@ abstract class ActiveRecord extends \yii\db\ActiveRecord
*/
public static function deleteAll($condition = null, $params = array())
{
$db = static::getDb();
$attributeKeys = array();
$pks = static::fetchPks($condition);
$db->executeCommand('MULTI');
foreach($pks as $pk) {
$pk = static::buildKey($pk);
$db->executeCommand('LREM', array(static::tableName(), 0, $pk));
$attributeKeys[] = static::tableName() . ':a:' . $pk;
// TODO use delete By Query feature
// http://www.elasticsearch.org/guide/reference/api/delete-by-query/
if (count($condition) != 1 || !isset($condition[reset(static::primaryKey())])) {
throw new NotSupportedException('DeleteAll is only supported by primary key in elasticsearch.');
}
if (empty($attributeKeys)) {
$db->executeCommand('EXEC');
return 0;
$query = '';
foreach((array) reset($condition) as $pk) {
if (is_array($pk)) {
$pk = reset($pk);
}
$query .= Json::encode(array(
"delete" => array(
"_id" => $pk,
"_type" => static::indexType(),
"_index" => static::indexName(),
),
)) . "\n";
}
$db->executeCommand('DEL', $attributeKeys);
$result = $db->executeCommand('EXEC');
return end($result);
$url = '/' . static::indexName() . '/' . static::indexType() . '/_bulk';
$response = static::getDb()->http()->post($url, array(), $query)->send();
$body = Json::decode($response->getBody(true));
$n=0;
foreach($body['items'] as $item) {
if ($item['delete']['ok']) {
$n++;
}
}
return $n;
}
/**
* Creates an [[ActiveQuery]] instance.
* This method is called by [[find()]], [[findBySql()]] and [[count()]] to start a SELECT query.
@@ -189,16 +194,6 @@ abstract class ActiveRecord extends \yii\db\ActiveRecord
return static::getTableSchema()->name;
}
/**
* This method is ment to be overridden in redis ActiveRecord subclasses to return a [[RecordSchema]] instance.
* @return RecordSchema
* @throws \yii\base\InvalidConfigException
*/
public static function getRecordSchema()
{
throw new InvalidConfigException(__CLASS__.'::getRecordSchema() needs to be overridden in subclasses and return a RecordSchema.');
}
public static function primaryKey()
{
return array('id');

31
framework/yii/elasticsearch/Connection.php
View File

@@ -160,27 +160,18 @@ class Connection extends Component
// TODO HTTP request to localhost:9200/
}
public function getQueryBuilder()
{
return new QueryBuilder($this);
}
/**
* @return \Guzzle\Http\Client
*/
public function http()
{
return new \Guzzle\Http\Client('http://localhost:9200/');
}
public function get($url)
{
$c = $this->initCurl($url);
$result = curl_exec($c);
curl_close($c);
}
private function initCurl($url)
{
$c = curl_init('http://localhost:9200/' . $url);
$fp = fopen("example_homepage.txt", "w");
curl_setopt($c, CURLOPT_FOLLOWLOCATION, false);
curl_setopt($c, CURLOPT_FILE, $fp);
curl_setopt($c, CURLOPT_HEADER, 0);
$guzzle = new \Guzzle\Http\Client('http://localhost:9200/');
//$guzzle->setDefaultOption()
return $guzzle;
}
}

18
framework/yii/elasticsearch/Query.php Normal file
View File

@@ -0,0 +1,18 @@
<?php
/**
* Created by JetBrains PhpStorm.
* User: cebe
* Date: 30.09.13
* Time: 11:39
* To change this template use File | Settings | File Templates.
*/
namespace yii\elasticsearch;
use yii\base\Component;
class Query extends Component
{
}

379
framework/yii/elasticsearch/QueryBuilder.php Normal file
View File

@@ -0,0 +1,379 @@
<?php
/**
* @link http://www.yiiframework.com/
* @copyright Copyright (c) 2008 Yii Software LLC
* @license http://www.yiiframework.com/license/
*/
namespace yii\elasticsearch;
use yii\base\NotSupportedException;
/**
* QueryBuilder builds a SELECT SQL statement based on the specification given as a [[Query]] object.
*
* QueryBuilder can also be used to build SQL statements such as INSERT, UPDATE, DELETE, CREATE TABLE,
* from a [[Query]] object.
*
* @author Qiang Xue <qiang.xue@gmail.com>
* @since 2.0
*/
class QueryBuilder extends \yii\base\Object
{
/**
* @var Connection the database connection.
*/
public $db;
/**
* Constructor.
* @param Connection $connection the database connection.
* @param array $config name-value pairs that will be used to initialize the object properties
*/
public function __construct($connection, $config = array())
{
$this->db = $connection;
parent::__construct($config);
}
/**
* Generates a SELECT SQL statement from a [[Query]] object.
* @param Query $query the [[Query]] object from which the SQL statement will be generated
* @return array the generated SQL statement (the first array element) and the corresponding
* parameters to be bound to the SQL statement (the second array element).
*/
public function build($query)
{
$searchQuery = array();
$this->buildSelect($searchQuery, $query->select);
// $this->buildFrom(&$searchQuery, $query->from);
$this->buildCondition($searchQuery, $query->where);
$this->buildOrderBy($searchQuery, $query->orderBy);
$this->buildLimit($searchQuery, $query->limit, $query->offset);
return $searchQuery;
}
/**
* Converts an abstract column type into a physical column type.
* The conversion is done using the type map specified in [[typeMap]].
* The following abstract column types are supported (using MySQL as an example to explain the corresponding
* physical types):
*
* - `pk`: an auto-incremental primary key type, will be converted into "int(11) NOT NULL AUTO_INCREMENT PRIMARY KEY"
* - `bigpk`: an auto-incremental primary key type, will be converted into "bigint(20) NOT NULL AUTO_INCREMENT PRIMARY KEY"
* - `string`: string type, will be converted into "varchar(255)"
* - `text`: a long string type, will be converted into "text"
* - `smallint`: a small integer type, will be converted into "smallint(6)"
* - `integer`: integer type, will be converted into "int(11)"
* - `bigint`: a big integer type, will be converted into "bigint(20)"
* - `boolean`: boolean type, will be converted into "tinyint(1)"
* - `float``: float number type, will be converted into "float"
* - `decimal`: decimal number type, will be converted into "decimal"
* - `datetime`: datetime type, will be converted into "datetime"
* - `timestamp`: timestamp type, will be converted into "timestamp"
* - `time`: time type, will be converted into "time"
* - `date`: date type, will be converted into "date"
* - `money`: money type, will be converted into "decimal(19,4)"
* - `binary`: binary data type, will be converted into "blob"
*
* If the abstract type contains two or more parts separated by spaces (e.g. "string NOT NULL"), then only
* the first part will be converted, and the rest of the parts will be appended to the converted result.
* For example, 'string NOT NULL' is converted to 'varchar(255) NOT NULL'.
*
* For some of the abstract types you can also specify a length or precision constraint
* by prepending it in round brackets directly to the type.
* For example `string(32)` will be converted into "varchar(32)" on a MySQL database.
* If the underlying DBMS does not support these kind of constraints for a type it will
* be ignored.
*
* If a type cannot be found in [[typeMap]], it will be returned without any change.
* @param string $type abstract column type
* @return string physical column type.
*/
public function getColumnType($type)
{
if (isset($this->typeMap[$type])) {
return $this->typeMap[$type];
} elseif (preg_match('/^(\w+)\((.+?)\)(.*)$/', $type, $matches)) {
if (isset($this->typeMap[$matches[1]])) {
return preg_replace('/\(.+\)/', '(' . $matches[2] . ')', $this->typeMap[$matches[1]]) . $matches[3];
}
} elseif (preg_match('/^(\w+)\s+/', $type, $matches)) {
if (isset($this->typeMap[$matches[1]])) {
return preg_replace('/^\w+/', $this->typeMap[$matches[1]], $type);
}
}
return $type;
}
/**
* @param array $columns
* @param boolean $distinct
* @param string $selectOption
* @return string the SELECT clause built from [[query]].
*/
public function buildSelect(&$query, $columns)
{
if (empty($columns)) {
return;
}
foreach ($columns as $i => $column) {
if (is_object($column)) {
$columns[$i] = (string)$column;
}
}
$query['fields'] = $columns;
}
/**
* @param array $columns
* @return string the ORDER BY clause built from [[query]].
*/
public function buildOrderBy(&$query, $columns)
{
if (empty($columns)) {
return;
}
$orders = array();
foreach ($columns as $name => $direction) {
// allow elasticsearch extended syntax as described in http://www.elasticsearch.org/guide/reference/api/search/sort/
if (is_array($direction)) {
$orders[] = array($name => $direction);
} elseif (is_string($direction)) {
$orders[] = $direction;
} else {
$orders[] = array($name => ($direction === Query::SORT_DESC ? 'desc' : 'asc'));
}
}
$query['sort'] = $orders;
}
/**
* @param integer $limit
* @param integer $offset
* @return string the LIMIT and OFFSET clauses built from [[query]].
*/
public function buildLimit(&$query, $limit, $offset)
{
if ($limit !== null && $limit >= 0) {
$query['size'] = $limit;
}
if ($offset > 0) {
$query['from'] = (int) $offset;
}
}
/**
* Parses the condition specification and generates the corresponding SQL expression.
* @param string|array $condition the condition specification. Please refer to [[Query::where()]]
* on how to specify a condition.
* @param array $params the binding parameters to be populated
* @return string the generated SQL expression
* @throws \yii\db\Exception if the condition is in bad format
*/
public function buildCondition(&$query, $condition)
{
static $builders = array(
'AND' => 'buildAndCondition',
'OR' => 'buildAndCondition',
'BETWEEN' => 'buildBetweenCondition',
'NOT BETWEEN' => 'buildBetweenCondition',
'IN' => 'buildInCondition',
'NOT IN' => 'buildInCondition',
'LIKE' => 'buildLikeCondition',
'NOT LIKE' => 'buildLikeCondition',
'OR LIKE' => 'buildLikeCondition',
'OR NOT LIKE' => 'buildLikeCondition',
);
if (empty($condition)) {
return;
}
if (!is_array($condition)) {
throw new NotSupportedException('String conditions are not supported by elasticsearch.');
}
if (isset($condition[0])) { // operator format: operator, operand 1, operand 2, ...
$operator = strtoupper($condition[0]);
if (isset($builders[$operator])) {
$method = $builders[$operator];
array_shift($condition);
$this->$method($query, $operator, $condition);
} else {
throw new Exception('Found unknown operator in query: ' . $operator);
}
} else { // hash format: 'column1' => 'value1', 'column2' => 'value2', ...
$this->buildHashCondition($query, $condition);
}
}
private function buildHashCondition(&$query, $condition)
{
$query['query']['term'] = $condition;
return; // TODO more
$parts = array();
foreach ($condition as $column => $value) {
if (is_array($value)) { // IN condition
$parts[] = $this->buildInCondition('IN', array($column, $value), $params);
} else {
if ($value === null) {
$parts[] = "$column IS NULL"; // TODO null
} elseif ($value instanceof Expression) {
$parts[] = "$column=" . $value->expression;
foreach ($value->params as $n => $v) {
$params[$n] = $v;
}
} else {
$phName = self::PARAM_PREFIX . count($params);
$parts[] = "$column=$phName";
$params[$phName] = $value;
}
}
}
return count($parts) === 1 ? $parts[0] : '(' . implode(') AND (', $parts) . ')';
}
private function buildAndCondition($operator, $operands, &$params)
{
$parts = array();
foreach ($operands as $operand) {
if (is_array($operand)) {
$operand = $this->buildCondition($operand, $params);
}
if ($operand !== '') {
$parts[] = $operand;
}
}
if (!empty($parts)) {
return '(' . implode(") $operator (", $parts) . ')';
} else {
return '';
}
}
private function buildBetweenCondition($operator, $operands, &$params)
{
if (!isset($operands[0], $operands[1], $operands[2])) {
throw new Exception("Operator '$operator' requires three operands.");
}
list($column, $value1, $value2) = $operands;
if (strpos($column, '(') === false) {
$column = $this->db->quoteColumnName($column);
}
$phName1 = self::PARAM_PREFIX . count($params);
$params[$phName1] = $value1;
$phName2 = self::PARAM_PREFIX . count($params);
$params[$phName2] = $value2;
return "$column $operator $phName1 AND $phName2";
}
private function buildInCondition($operator, $operands, &$params)
{
if (!isset($operands[0], $operands[1])) {
throw new Exception("Operator '$operator' requires two operands.");
}
list($column, $values) = $operands;
$values = (array)$values;
if (empty($values) || $column === array()) {
return $operator === 'IN' ? '0=1' : '';
}
if (count($column) > 1) {
return $this->buildCompositeInCondition($operator, $column, $values, $params);
} elseif (is_array($column)) {
$column = reset($column);
}
foreach ($values as $i => $value) {
if (is_array($value)) {
$value = isset($value[$column]) ? $value[$column] : null;
}
if ($value === null) {
$values[$i] = 'NULL';
} elseif ($value instanceof Expression) {
$values[$i] = $value->expression;
foreach ($value->params as $n => $v) {
$params[$n] = $v;
}
} else {
$phName = self::PARAM_PREFIX . count($params);
$params[$phName] = $value;
$values[$i] = $phName;
}
}
if (strpos($column, '(') === false) {
$column = $this->db->quoteColumnName($column);
}
if (count($values) > 1) {
return "$column $operator (" . implode(', ', $values) . ')';
} else {
$operator = $operator === 'IN' ? '=' : '<>';
return "$column$operator{$values[0]}";
}
}
protected function buildCompositeInCondition($operator, $columns, $values, &$params)
{
$vss = array();
foreach ($values as $value) {
$vs = array();
foreach ($columns as $column) {
if (isset($value[$column])) {
$phName = self::PARAM_PREFIX . count($params);
$params[$phName] = $value[$column];
$vs[] = $phName;
} else {
$vs[] = 'NULL';
}
}
$vss[] = '(' . implode(', ', $vs) . ')';
}
foreach ($columns as $i => $column) {
if (strpos($column, '(') === false) {
$columns[$i] = $this->db->quoteColumnName($column);
}
}
return '(' . implode(', ', $columns) . ") $operator (" . implode(', ', $vss) . ')';
}
private function buildLikeCondition($operator, $operands, &$params)
{
if (!isset($operands[0], $operands[1])) {
throw new Exception("Operator '$operator' requires two operands.");
}
list($column, $values) = $operands;
$values = (array)$values;
if (empty($values)) {
return $operator === 'LIKE' || $operator === 'OR LIKE' ? '0=1' : '';
}
if ($operator === 'LIKE' || $operator === 'NOT LIKE') {
$andor = ' AND ';
} else {
$andor = ' OR ';
$operator = $operator === 'OR LIKE' ? 'LIKE' : 'NOT LIKE';
}
if (strpos($column, '(') === false) {
$column = $this->db->quoteColumnName($column);
}
$parts = array();
foreach ($values as $value) {
$phName = self::PARAM_PREFIX . count($params);
$params[$phName] = $value;
$parts[] = "$column $operator $phName";
}
return implode($andor, $parts);
}
}