`yf_search_driver_sphinxsearch` class

Sphinx 3.x over SphinxQL (Manticore: the manticore driver extends this one). Indexes are RT indexes declared in sphinx.conf (Sphinx has no CREATE TABLE); highlight needs the fields in stored_fields, suggest needs min_infix_len or min_prefix_len. Statements go through yf_sphinxsearch::execute() (no cache) on a connection of the driver's own. Escaping: MATCH text with escape() (ASCII punctuation backslashed, operator words lowercased, then the string literal), other strings with _literal(), numbers by PHP type. A filter value keeps its PHP type: SphinxQL compares an int attribute only with numbers and a string attribute only with strings, so cast request data to the attribute's type. Names (index, fields, filters, sort, facets, return) must come from the caller's whitelist, never from request data: they are checked as identifiers but sent bare (Sphinx 3 rejects a backticked string attribute in WHERE), so a SphinxQL keyword used as a name is a syntax error.

File	`plugins/search/classes/search/yf_search_driver_sphinxsearch.class.php`
Use	`_class('search_driver_sphinxsearch', 'classes/search/')`
Extends	`yf_search_driver`

Public properties

A site sets them in its config with $PROJECT_CONF['search_driver_sphinxsearch']['NAME'] = ...;

Property	Default	Description
`$BULK_SIZE` int	`500`	Documents per REPLACE statement in bulk()
`$CONNECT_TIMEOUT` int\|null	`yf_sphinxsearch's */ Property [ public $CONNECT_TIMEOUT = NULL`	In seconds, null = yf_sphinxsearch's
`$HOST` string	`the yf_sphinxsearch HOST (SPHINX_HOST / SPHINX_PORT or its project conf) */ Property [ public $HOST = ''`	host:port of the SphinxQL listener, '' = the yf_sphinxsearch HOST (SPHINX_HOST / SPHINX_PORT or its project conf)
`$PARENT`	`null`
`$RECONNECT_TRIES` int\|null	`yf_sphinxsearch's */ Property [ public $RECONNECT_TRIES = NULL`	null = yf_sphinxsearch's
`$SPHINX` object\|null	`null`	yf_sphinxsearch transport, built on first use
`$SUGGEST_LIMIT` int	`10`	Default number of suggest() results

Inherited properties

from yf_search_driver: $DEF_LIMIT, $MAX_LIMIT, $MAX_WINDOW, $MAX_WORDS, $last_error

Public methods 28

add

public function add($index, $id, $doc)

Insert or replace one document (see bulk()).

    public function add($index, $id, $doc)
    {
        return $this->bulk($index, [$id => $doc]);
    }

bulk

public function bulk($index, $docs)

REPLACE, one statement per BULK_SIZE documents with the same fields; a failing statement stops the rest, earlier ones stay written. Array values are stored as JSON, null values are left out (the field gets its default).

$docs mixed id => doc

    public function bulk($index, $docs)
    {
        if ( ! $this->_is_valid_name($index)) {
            return $this->_fail('bad index');
        }
        $groups = [];
        foreach ((array) $docs as $id => $doc) {
            $doc_id = $this->_doc_id($id);
            $doc = $this->_without_id(array_filter((array) $doc, fn ($v) => $v !== null));
            if ($doc_id === false || ! $doc) {
                return $this->_fail('bad document ' . ($doc_id === false ? 'id' : $doc_id));
            }
            $values = [$doc_id];
            foreach ($doc as $name => $value) {
                if ( ! $this->_is_valid_name($name)) {
                    return $this->_fail('bad field name in document ' . $doc_id);
                }
                $sql_value = $this->_value($value);
                if ($sql_value === false) {
                    return $this->_fail('bad value of ' . $name . ' in document ' . $doc_id);
                }
                $values[] = $sql_value;
            }
            $groups[implode(',', array_keys($doc))][] = '(' . implode(', ', $values) . ')';
        }
        $error = '';
        foreach ($groups as $names => $rows) {
            $columns = implode(', ', array_map([$this, '_quote_name'], ['id', ...explode(',', $names)]));
            foreach (array_chunk($rows, max(1, (int) $this->BULK_SIZE)) as $chunk) {
                $sql = 'REPLACE INTO ' . $this->_quote_name($index) . ' (' . $columns . ') VALUES ' . implode(', ', $chunk);
                if ($this->_execute($sql, $error) === false) {
                    return $this->_fail($error);
                }
            }
        }
        $this->last_error = '';
        return true;
    }

capabilities

public function capabilities()

    public function capabilities()
    {
        return ['highlight' => true, 'suggest' => true, 'facets' => true, 'write' => true, 'ddl' => false, 'json' => true];
    }

connect

public function connect()

    public function connect()
    {
        $sphinx = $this->_transport();
        return $sphinx->_connect() ? $sphinx : $this->_fail('cannot connect to ' . $sphinx->HOST);
    }

create_index

public function create_index($index, $schema = [])

Sphinx has no CREATE TABLE: an RT index exists only in sphinx.conf.

    public function create_index($index, $schema = [])
    {
        return $this->_fail('create_index() is not possible, declare the index in sphinx.conf (type = rt)');
    }

delete

public function delete($index, $ids)

$ids mixed one id or a list

    public function delete($index, $ids)
    {
        if ( ! $this->_is_valid_name($index)) {
            return $this->_fail('bad index');
        }
        $list = [];
        foreach ((array) $ids as $id) {
            $doc_id = $this->_doc_id($id);
            if ($doc_id === false) {
                return $this->_fail('bad id');
            }
            $list[$doc_id] = $doc_id;
        }
        return $list ? $this->_write('DELETE FROM ' . $this->_quote_name($index) . ' WHERE id IN (' . implode(', ', $list) . ')') : true;
    }

escape

public function escape($text)

Text for MATCH('...'): query syntax and string literal escaped.

    public function escape($text)
    {
        return $this->_literal($this->_match_escape($text));
    }

index_exists

public function index_exists($index)

    public function index_exists($index)
    {
        if ( ! $this->_is_valid_name($index)) {
            return $this->_fail('bad index');
        }
        $error = '';
        $result = $this->_execute("SHOW TABLES LIKE '" . $this->_literal($index) . "'", $error);
        if ($result === false) {
            return $this->_fail($error);
        }
        $this->last_error = '';
        $column = $this->_tables_column();
        foreach ($result['rows'] as $row) {
            if (strtolower((string) ($row[$column] ?? '')) === strtolower($index)) {
                return true;
            }
        }
        return false;
    }

is_ready

public function is_ready()

    public function is_ready()
    {
        return (bool) $this->connect();
    }

raw

public function raw($query)

SphinxQL passthrough: rows of the first result set ([] for a write). Nothing is escaped: trusted SQL only, never request data.

    public function raw($query)
    {
        $error = '';
        $result = $this->_execute((string) $query, $error);
        if ($result === false) {
            return $this->_fail($error);
        }
        $this->last_error = '';
        return $result['rows'];
    }

search

public function search(array $params = [], &$error_message = '')

$params array

    public function search(array $params = [], &$error_message = '')
    {
        $started = microtime(true);
        $error_message = $error = '';
        $p = $this->_normalize_params($params, $error);
        if ($p !== false && ! $this->_is_valid_name($p['index'])) {
            [$p, $error] = [false, 'bad or missing index'];
        }
        if ($p === false) {
            return $this->_fail($error, $error_message);
        }
        $markers = $this->_highlight_markers();
        $result = $this->_execute($this->_search_sql($p, $markers), $error);
        $meta = $result === false ? false : $this->_execute('SHOW META', $error);
        if ($meta === false) {
            return $this->_fail($error, $error_message);
        }
        $meta = array_column($meta['rows'], 'Value', 'Variable_name');
        $items = [];
        foreach ($result['rows'] as $row) {
            $item = ['id' => $row[$p['id_field']] ?? null, 'score' => (float) ($row['_score'] ?? 0), 'doc' => [], 'highlight' => []];
            foreach ($p['highlight']['fields'] ?? [] as $name) {
                $item['highlight'][$name] = $this->_highlight_html((string) ($row['_hl_' . $name] ?? ''), $markers, $p['highlight']['pre'], $p['highlight']['post']);
                unset($row['_hl_' . $name]);
            }
            unset($row['_score']);
            $item['doc'] = $row;
            $items[] = $item;
        }
        $facets = [];
        foreach ($p['facets'] as $i => $name) {
            $facets[$name] = [];
            foreach ($result['sets'][$i] ?? [] as $row) {
                $facets[$name][] = ['value' => $row[$name] ?? null, 'count' => (int) ($row['count(*)'] ?? 0)];
            }
        }
        $this->last_error = '';
        return [
            'total' => (int) ($meta['total_found'] ?? count($items)),
            'total_relation' => ($meta['total_relation'] ?? 'eq') === 'gte' ? 'gte' : 'eq',
            'items' => $items,
            'facets' => $facets,
            'took_ms' => round((microtime(true) - $started) * 1000, 2),
        ];
    }

suggest

public function suggest($text, $params = [])

Completions of the last word of $text, each after the words before it: CALL KEYWORDS('word*'), needs min_prefix_len or min_infix_len; most documents first. 'mode' => 'fuzzy': spelling corrections of the last word instead (CALL QSUGGEST, needs min_infix_len).

$params mixed index, limit, mode

    public function suggest($text, $params = [])
    {
        $params = (array) $params;
        if ( ! $this->_is_valid_name($params['index'] ?? '')) {
            return $this->_fail('suggest needs index');
        }
        $words = $this->_words($text);
        if ( ! $words) {
            return [];
        }
        $last = array_pop($words);
        $before = $words ? implode(' ', $words) . ' ' : '';
        $limit = min(max(1, (int) $this->MAX_LIMIT), max(1, (int) ($params['limit'] ?? $this->SUGGEST_LIMIT)));
        $index = "'" . $this->_literal($params['index']) . "'";
        $error = '';
        if (($params['mode'] ?? '') === 'fuzzy') {
            $result = $this->_execute("CALL QSUGGEST('" . $this->_literal($last) . "', " . $index . ', ' . $limit . ' AS limit)', $error);
            if ($result === false) {
                return $this->_fail($error);
            }
            $this->last_error = '';
            return array_map(fn ($row) => $before . $row['suggest'], $result['rows']);
        }
        $result = $this->_execute("CALL KEYWORDS('" . $this->escape($last) . "*', " . $index . ', 0 AS fold_wildcards, 1 AS stats)', $error);
        if ($result === false) {
            return $this->_fail($error);
        }
        $docs = [];
        foreach ($result['rows'] as $row) {
            // tokenized is the prefix after the index's own case and accent folding
            $tokenized = (string) ($row['tokenized'] ?? '');
            $word = (string) ($row['normalized'] ?? '');
            if (str_ends_with($tokenized, '*') && ! str_contains($word, '*') && str_starts_with($word, substr($tokenized, 0, -1))) {
                $docs[$word] = max($docs[$word] ?? 0, (int) ($row['docs'] ?? 0));
            }
        }
        uksort($docs, fn ($a, $b) => [$docs[$b], $a] <=> [$docs[$a], $b]);
        $this->last_error = '';
        return array_map(fn ($word) => $before . $word, array_slice(array_keys($docs), 0, $limit));
    }

truncate

public function truncate($index)

    public function truncate($index)
    {
        return $this->_is_valid_name($index) ? $this->_write('TRUNCATE RTINDEX ' . $this->_quote_name($index)) : $this->_fail('bad index');
    }

update

public function update($index, $id, $fields)

Attributes only: a full-text field cannot be UPDATEd, add() the whole document instead.

    public function update($index, $id, $fields)
    {
        $doc_id = $this->_doc_id($id);
        $fields = $this->_without_id((array) $fields);
        if ( ! $this->_is_valid_name($index) || $doc_id === false || ! $fields) {
            return $this->_fail('update needs index, id and fields');
        }
        $set = [];
        foreach ($fields as $name => $value) {
            $sql_value = $value === null ? false : $this->_value($value);
            if ( ! $this->_is_valid_name($name) || $sql_value === false) {
                return $this->_fail('bad field ' . ($this->_is_valid_name($name) ? $name : 'name'));
            }
            $set[] = $this->_quote_name($name) . ' = ' . $sql_value;
        }
        return $this->_write('UPDATE ' . $this->_quote_name($index) . ' SET ' . implode(', ', $set) . ' WHERE id = ' . $doc_id);
    }

Methods starting with `_` hooks and helpers; not routable as page actions

__call

public function __call($name, $args)

Catch missing method call.

    public function __call($name, $args)
    {
        return main()->extend_call($this, $name, $args);
    }

_execute

public function _execute($sql, &$error = '')

Returns array|false the execute() result, or false with the reason in $error

    public function _execute($sql, &$error = '')
    {
        $result = $this->_transport()->execute($sql);
        if ($result['error'] !== '') {
            $error = 'query failed: ' . $result['error'];
            return false;
        }
        return $result;
    }

_filters_sql

public function _filters_sql($filters)

$filters mixed normalized filters

Returns string[]

    public function _filters_sql($filters)
    {
        $sql = [];
        foreach ((array) $filters as $name => $conds) {
            $col = $this->_quote_name($name);
            foreach ($conds as [$op, $value]) {
                if ($op === 'in' || $op === 'not_in') {
                    $sql[] = $col . ($op === 'in' ? ' IN (' : ' NOT IN (') . implode(', ', array_map([$this, '_value'], $value)) . ')';
                } else {
                    $sql[] = $col . ' ' . $op . ' ' . $this->_value($value);
                }
            }
        }
        return $sql;
    }

_highlight_sql

public function _highlight_sql($name, $pre, $post)

Whole stored field with the markers around the matches (needs stored_fields).

$name mixed validated field name
$pre mixed marker from _highlight_markers(), not HTML

    public function _highlight_sql($name, $pre, $post)
    {
        return 'SNIPPET(' . $this->_quote_name($name) . ", QUERY(), 'before_match=" . $this->_literal($pre) . "', 'after_match=" . $this->_literal($post) . "', 'limit=0')";
    }

_init

public function _init()

Kept empty for site subclasses that call parent::_init(); it no longer loads send_mail.

    public function _init()
    {
    }

_literal

public function _literal($text)

Inside '...': the SphinxQL string parser knows only \\ and \'.

    public function _literal($text)
    {
        return strtr(str_replace("\0", '', (string) $text), ['\\' => '\\\\', "'" => "\\'"]);
    }

_match_escape

public function _match_escape($text)

Query syntax only: every ASCII punctuation character gets a backslash (the query parser reads \x as a plain x), control characters become spaces, and the operator words, which are uppercase only, are lowercased.

    public function _match_escape($text)
    {
        $text = preg_replace('~[\x00-\x1f\x7f]~', ' ', (string) $text);
        $text = preg_replace('~[!-/:-@\[-`{-\~]~', '\\\\$0', $text);
        $operators = '~(?<![A-Za-z0-9_])(MAYBE|NEAR|NOTNEAR|SENTENCE|PARAGRAPH|ZONE|ZONESPAN)(?![A-Za-z0-9_])~';
        return preg_replace_callback($operators, fn ($m) => strtolower($m[1]), $text);
    }

_options

public function _options($p, $words)

Items of the OPTION clause.

$p mixed normalized params

Returns string[]

    public function _options($p, $words)
    {
        if ( ! $p['fields'] || ! $words) {
            return [];
        }
        $weights = [];
        foreach ($p['fields'] as $name => $weight) {
            // field_weights takes positive integers only
            $weights[] = $name . '=' . max(1, (int) round($weight));
        }
        return ['field_weights=(' . implode(', ', $weights) . ')'];
    }

_quote_name

public function _quote_name($name)

Names are validated by _normalize_params() / _is_valid_name() and stay bare: Sphinx 3 rejects a backticked string attribute in WHERE. A name that is a SphinxQL keyword is a syntax error.

    public function _quote_name($name)
    {
        return preg_replace('~[^a-z0-9_]~i', '', (string) $name);
    }

_search_sql

public function _search_sql($p, $markers = ['', ''])

SELECT for normalized params: MATCH, filters, order, limit, options and one FACET per facet field.

$markers mixed from _highlight_markers()

    public function _search_sql($p, $markers = ["\x02", "\x03"])
    {
        $words = $this->_words($p['q']);
        $where = $this->_filters_sql($p['filters']);
        if ($words) {
            $match = implode(' ', array_map([$this, 'escape'], $words));
            $fields = $p['fields'] ? '@(' . implode(',', array_keys($p['fields'])) . ') ' : '';
            array_unshift($where, "MATCH('" . $fields . $match . "')");
        }
        $columns = '*';
        if ($p['return']) {
            $columns = implode(', ', array_map([$this, '_quote_name'], array_unique(['id', $p['id_field'], ...$p['return']])));
        }
        $columns .= ', WEIGHT() AS _score';
        foreach ($p['highlight']['fields'] ?? [] as $name) {
            $columns .= ', ' . $this->_highlight_sql($name, $markers[0], $markers[1]) . ' AS ' . $this->_quote_name('_hl_' . $name);
        }
        $order = [];
        $sort = $p['sort'] ?: ($words ? ['_score' => 'desc'] : []);
        foreach ($sort as $name => $dir) {
            $order[] = ($name === '_score' ? 'WEIGHT()' : $this->_quote_name($name)) . ' ' . strtoupper($dir);
        }
        // Sphinx sorts by at most 5 keys
        if ( ! isset($sort[$p['id_field']]) && count($order) < 5) {
            $order[] = $this->_quote_name($p['id_field']) . ' ASC';
        }
        $sql = 'SELECT ' . $columns . ' FROM ' . $this->_quote_name($p['index'])
            . ($where ? ' WHERE ' . implode(' AND ', $where) : '')
            . ' ORDER BY ' . implode(', ', $order) . ' LIMIT ' . (int) $p['offset'] . ', ' . (int) $p['limit'];
        $options = $this->_options($p, $words);
        if ($options) {
            $sql .= ' OPTION ' . implode(', ', $options);
        }
        foreach ($p['facets'] as $name) {
            $col = $this->_quote_name($name);
            $sql .= ' FACET ' . $col . ' ORDER BY COUNT(*) DESC, ' . $col . ' ASC LIMIT ' . (int) $p['facet_limit'];
        }
        return $sql;
    }

_tables_column

public function _tables_column()

Name column of SHOW TABLES.

    public function _tables_column()
    {
        return 'Index';
    }

_transport

public function _transport()

A yf_sphinxsearch of the driver's own, set up from the driver's props or, when they are empty, the shared instance's.

    public function _transport()
    {
        if ( ! is_object($this->SPHINX)) {
            $shared = _class('sphinxsearch');
            $class = get_class($shared);
            $sphinx = new $class();
            foreach (['HOST', 'CONNECT_TIMEOUT', 'RECONNECT_TRIES', 'RECONNECT_SLEEP', 'QUERY_RETRY_SLEEP'] as $name) {
                $own = property_exists($this, $name) ? $this->$name : null;
                $sphinx->$name = ($own === null || $own === '') ? $shared->$name : $own;
            }
            $this->SPHINX = $sphinx;
        }
        return $this->SPHINX;
    }

_value

public function _value($value)

SphinxQL literal by PHP type: ints and bools unquoted, floats in their shortest round-trip form (1.0E+20 is accepted), arrays as JSON, the rest escaped and quoted; false for INF / NAN or what JSON cannot encode.

Returns string|false

    public function _value($value)
    {
        if (is_int($value) || is_bool($value)) {
            return (string) (int) $value;
        }
        if (is_float($value)) {
            return is_finite($value) ? var_export($value, true) : false;
        }
        if (is_array($value) || is_object($value)) {
            $value = json_encode($value, JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES | JSON_INVALID_UTF8_SUBSTITUTE);
            if ($value === false) {
                return false;
            }
        }
        return "'" . $this->_literal((string) $value) . "'";
    }

_write

public function _write($sql)

    public function _write($sql)
    {
        $error = '';
        if ($this->_execute($sql, $error) === false) {
            return $this->_fail($error);
        }
        $this->last_error = '';
        return true;
    }

Inherited methods

from yf_search_driver: _doc_id(), _fail(), _highlight_html(), _highlight_markers(), _is_valid_name(), _name(), _normalize_filter(), _normalize_params(), _not_supported(), _without_id(), _words(), drop_index()

yf_search_driver_sphinxsearch class