File: /var/www/limestate-admin/app/Services/ElasticIndexService.php
<?php
namespace App\Services;
use App\Collections\BuildingCollection;
use App\Collections\ComplexCollection;
use App\Collections\FlatCollection;
use App\Models\Building;
use App\Models\Complex;
use App\Models\Flat;
use App\Models\FlatOption;
use App\Repositories\ElasticRepository;
use Elastic\Elasticsearch\Client;
use Elastic\Elasticsearch\ClientBuilder;
use Elastic\Elasticsearch\Exception\ClientResponseException;
use Illuminate\Support\Facades\DB;
use Illuminate\Support\Facades\Storage;
use Illuminate\Support\Str;
class ElasticIndexService
{
public const NORMAL = "\033[0m";
public const FONT_RED = "\033[31m\033[1m";
public const FONT_GREEN = "\033[32m\033[1m";
public const FONT_YELLOW = "\033[33m\033[1m";
public const INDEX_COMPLEXES = 'complexes';
public const INDEX_SECOND = 'second';
public const INDEX_FLATS = 'flats';
private ?string $host;
private ?Client $client = null;
private bool $isCli = false;
public bool $withElastic = true;
public array $messages = [];
private array $currentIndices = [];
private ?string $newIndex = null;
// MARK: - Output
private function echo(string $msg, bool $endLine = true, string $color = ''): void
{
if ($this->isCli) {
print_r($color . $msg . ($endLine ? PHP_EOL : '') . self::NORMAL);
} else {
$this->messages[] = $msg;
}
}
// MARK: - Helpers
private function initElastic(): bool
{
if (!$this->withElastic) {
$this->echo('Запуск без эластика', color: self::FONT_YELLOW);
return false;
}
if (empty($this->host)) {
$this->echo('Хост не указан', color: self::FONT_RED);
return false;
}
$params = ['hosts' => [$this->host], 'retries' => 2];
$this->client = ClientBuilder::fromConfig($params, true);
return true;
}
private function clearIndex(?string $index): bool
{
$this->echo('Удаляем индекс: ', false);
if (is_null($index)) {
$this->echo('Индекс отсутствует', color: self::FONT_YELLOW);
return true;
}
$this->echo($index, color: self::FONT_GREEN);
$indexExists = $this->client->indices()->exists(['index' => $index])->asBool();
if ($indexExists) {
$params = ['index' => $index];
try {
$response = $this->client->indices()->delete($params);
$this->echo($response, color: self::FONT_GREEN);
$this->echo('Удалили индекс');
} catch (ClientResponseException) {
$this->echo('Индекс отсутствует', color: self::FONT_YELLOW);
}
}
return true;
}
private function createIndex(string $index, string $scheme): bool
{
$this->echo('Создаём индекс');
$indexPath = resource_path('indexes/' . $scheme . '.json');
if (!file_exists($indexPath)) {
$this->echo('Схема не найдена');
return false;
}
$indexExists = $this->client->indices()->exists(['index' => $index])->asBool();
if (!$indexExists) {
$indexSchema = json_decode(file_get_contents($indexPath), true, 512, JSON_THROW_ON_ERROR);
$params = ['index' => $index, 'body' => $indexSchema];
$this->client->indices()->create($params);
$this->echo('Индекс создан' . PHP_EOL, color: self::FONT_GREEN);
}
return true;
}
private function send(string $index, array $indexData): void
{
if (!$this->withElastic) {
Storage::disk('public')->put($index . '.json', json_encode($indexData, JSON_THROW_ON_ERROR | JSON_PRETTY_PRINT | JSON_UNESCAPED_UNICODE));
return;
}
$params = ['index' => $index, 'body' => $indexData];
$this->client->index($params);
}
private function initAlias(string $alias, string $newIndex): void
{
$aliasExists = $this->client->indices()->existsAlias(['name' => $alias])->asBool();
if ($aliasExists) {
$this->echo('Alias с именем ' . $alias . ' уже существует', color: self::FONT_GREEN);
return;
}
$indexExists = $this->client->indices()->exists(['index' => $alias])->asBool();
if ($indexExists) {
$this->clearIndex($alias);
}
$this->echo('Создаём alias ' . $alias);
$this->client->indices()->putAlias(['index' => $newIndex, 'name' => $alias]);
}
private function switchAlias(string $alias): void
{
$actions = [];
$hasAlias = $this->client->indices()->existsAlias(['name' => $alias])->asBool();
if ($hasAlias) {
$old = $this->client->indices()->getAlias(['name' => $alias])->asArray();
foreach (array_keys($old) as $oldIndex) {
$actions[] = ['remove' => ['index' => $oldIndex, 'alias' => $alias]];
}
} else {
$this->initAlias($alias, $this->newIndex);
}
$actions[] = ['add' => ['index' => $this->newIndex, 'alias' => $alias]];
$this->client->indices()->updateAliases(['body' => ['actions' => $actions]]);
}
// MARK: - Elastic methods
public function getIndicesList(): array
{
$this->initElastic();
return $this->client->cat()->indices(['format' => 'json'])->asArray();
}
public function getAliasesList(): array
{
$this->initElastic();
return $this->client->indices()->getAlias()->asArray();
}
public function clearIndices(array $indices): void
{
if (empty($indices)) {
return;
}
$this->initElastic();
foreach ($indices as $index) {
$this->clearIndex($index);
}
}
// MARK: - Core
public function __construct()
{
$this->host = config('elastic.host');
// $this->withElastic = in_array(App::environment(), ['prod', 'production', 'stage']);
$this->withElastic = true;
}
public function setIsCli(bool $isCli = true): void
{
$this->isCli = $isCli;
}
// MARK: - Proceed indices
private function prepareIndexing(string $index): bool
{
if (!$this->initElastic()) {
return false;
}
$indices = $this->getIndicesList();
$indices = array_column($indices, 'index');
$this->currentIndices = array_filter($indices, static fn($item) => str_contains($item, $index)) ?? [];
$this->newIndex = $index . '_' . Str::lower(Str::random());
if (!$this->clearIndex($this->newIndex)) {
return false;
}
if (!$this->createIndex($this->newIndex, $index)) {
return false;
}
return true;
}
public function complexes(): void
{
if (!$this->prepareIndexing(self::INDEX_COMPLEXES)) {
return;
}
$this->echo('Добавляем данные по комплексам в индекс' . PHP_EOL);
/** @var ComplexCollection $complexes */
$complexes = Complex::query()->where('published', 1)->orderBy('name')->cursor();
$totalCount = $complexes->count();
$this->echo('Всего комплексов: ' . $totalCount . PHP_EOL, color: self::FONT_GREEN);
foreach ($complexes as $complex) {
$this->echo(str_pad($complex->id, 5, ' ', STR_PAD_LEFT) . ': ' . $complex->name . ' (' . $complex->alias . ')', false);
if (empty($complex->buildings)) {
$this->echo(' нет зданий', self::FONT_YELLOW);
continue;
}
if (empty($complex->getFlatsCount())) {
$this->echo(' нет квартир', color: self::FONT_RED);
continue;
}
$isApartment = (boolean) $complex->is_apartment;
$complexAdditional = DB::table('complexes AS c')
->select(DB::raw('MIN(f.price) AS min_price, MAX(f.price) AS max_price, cd.name AS city_district, m.name AS metro, GROUP_CONCAT(DISTINCT(fo.alias) SEPARATOR ", ") AS options'))
->join('buildings AS b', 'b.complex_id', '=', 'c.id')
->join('flats AS f', 'f.building_id', '=', 'b.id')
->join('city_districts AS cd', 'cd.id', '=', 'c.city_district_id', 'left')
->join('metros AS m', 'm.id', '=', 'c.metro_id', 'left')
->join('flat_options_relations AS fro', 'f.id', '=', 'fro.flat_id', 'left')
->join('flat_options AS fo', 'fo.id', '=', 'fro.option_id', 'left')
->where('c.published', 1)
->where('b.published', 1)
->where('f.published', 1)
->whereNull('c.deleted_at')
->whereNull('b.deleted_at')
->whereNull('f.deleted_at')
->where('c.id', $complex->id)
->where('f.price', '>', 0)
->groupBy('c.id')
->first();
$complexData = [
'id' => $complex->id,
'name' => $complex->name,
'alias' => $complex->alias,
'description' => $complex->description ?? '',
'beautification' => $complex->beautification,
'infrastructure' => $complex->infrastructure,
'region_id' => $complex->region_id,
'region_district_id' => $complex->region_district_id,
'metro_id' => $complex->metro_id,
'city_id' => $complex->city_id,
'started_year' => $complex->started_year,
'started_quarter' => $complex->started_quarter,
'completed_year' => $complex->completed_year,
'completed_quarter' => $complex->completed_quarter,
'is_completed' => (boolean) $complex->is_completed,
'developer_id' => $complex->developer_id,
'latlng' => $complex->latlng,
'is_special' => (boolean) $complex->is_special,
'is_apartment' => $isApartment,
'special_offer' => $complex->special_offer,
'schema_coords' => $complex->schema_coords,
'min_price' => $complexAdditional->min_price ?? 0,
'max_price' => $complexAdditional->max_price ?? 0,
'street' => $complex->street_name ?? '',
'city_district' => $complexAdditional->city_district ?? '',
'city_district_id' => $complex->city_district_id,
'metro' => $complexAdditional->metro ?? '',
'options' => $complexAdditional->options ?? '',
'published' => (boolean) $complex->published,
'category_id' => $complex->category_id,
'gallery' => [],
'gallery_thumbs' => [],
'full_thumbs' => [],
];
$buildingsData = [];
foreach ($complex->buildings as $building) {
if (empty($building->flats)) {
continue;
}
$buildingData = [
'id' => $building->id,
'name' => $building->name,
'alias' => $building->alias,
'street_name' => $building->street_name,
'floors' => $building->floors,
'region_id' => $building->region_id,
'region_district_id' => $building->region_district_id,
'metro_id' => $building->metro_id,
'city_id' => $building->city_id,
'city_district_id' => $building->city_district_id,
'started_year' => $building->started_year,
'started_quarter' => $building->started_quarter,
'completed_year' => $building->completed_year,
'completed_quarter' => $building->completed_quarter,
'is_completed' => (boolean) $building->is_completed,
'developer_id' => $building->developer_id,
'latlng' => $building->latlng,
'is_special' => (boolean) $building->is_special,
'special_offer' => $building->special_offer,
'queue' => $building->queue,
'is_single' => (boolean) $building->is_single,
'housing_number' => $building->housing_number,
'is_own' => $building->is_own,
'street_type_id' => $building->street_type_id,
'house_number' => $building->house_number,
'house_letter' => $building->house_letter,
'entrance_count' => $building->entrance_count,
'flats_count' => $building->flats_count,
'rooms_count' => $building->rooms_count,
'finishing_type_id' => $building->finishing_type_id,
'house_type_id' => $building->house_type_id,
'orientation_id' => $building->orientation_id,
'published' => (boolean) $building->published,
'category_id' => $building->category_id,
'address' => $building->generateAddress() ?? '',
'is_apartment' => $isApartment,
];
$flatsData = $building->flats->map(fn(Flat $flat) => [
'id' => $flat->id,
'floor' => $flat->floor,
'rooms' => $flat->rooms,
'total_area' => $flat->total_area,
'living_area' => $flat->living_area,
'rooms_area' => $flat->rooms_area,
'kitchen_area' => $flat->kitchen_area,
'number' => $flat->number,
'price' => $flat->price,
'base_price' => $flat->base_price,
'mortage_price' => $flat->mortgage_price ?? 0,
'assigment' => $flat->assignment ?? '',
'contractor' => $flat->contractor,
'bathroom_type_id' => $flat->bathroom_type_id,
'balcony_type_id' => $flat->balcony_type_id,
'finishing_type_id' => $flat->finishing_type_id,
'view_type_id' => $flat->view_type_id,
'entrance' => $flat->entrance,
'rooms_area_separately' => $flat->rooms_area_separately,
'published' => (boolean) $flat->published,
'description' => $flat->description,
'options' => $flat->options->map(fn($option) => ['option_id' => $option->id])->toArray(),
'is_apartment' => $isApartment,
])->toArray();
if (empty($flatsData)) {
continue;
}
$buildingData['flats'] = $flatsData;
$buildingsData[] = $buildingData;
}
$attachments = ElasticRepository::getAllAttachments($complex);
ElasticRepository::fillGallery($attachments, $complexData, $complex->plan_id, true);
$this->echo(' попало', color: self::FONT_GREEN);
$complexData['buildings'] = $buildingsData;
$complexData['max_price'] = min(2147483646, $complexData['max_price']);
$this->send($this->newIndex, $complexData);
}
$this->echo('Добавление комплексов завершено');
$this->switchAlias(self::INDEX_COMPLEXES);
$this->clearIndices($this->currentIndices);
}
public function second(): void
{
if (!$this->prepareIndexing(self::INDEX_SECOND)) {
return;
}
$this->echo('Добавляем данные по вторичке в индекс');
$query = DB::table('buildings AS b')
->select(DB::raw('b.*, GROUP_CONCAT(DISTINCT(fo.alias) SEPARATOR ", ") AS options'))
->join('flats AS f', 'f.building_id', '=', 'b.id')
->join('flat_options_relations AS fro', 'f.id', '=', 'fro.flat_id', 'left')
->join('flat_options AS fo', 'fo.id', '=', 'fro.option_id', 'left')
->where('b.published', 1)
->where('f.is_secondary', true)
->groupBy('b.id');
/** @var BuildingCollection $buildings */
$buildings = (clone $query)->cursor();
$buildingsIds = $query->pluck('id');
$this->echo('Всего зданий: ' . count($buildingsIds));
$flatCount = Flat::query()
->whereIn('building_id', $buildingsIds)
->where('published', 1)
->where('is_secondary', true)
->count();
$this->echo('Всего квартир: ' . $flatCount);
foreach ($buildings as $buildingRecord) {
/** @var Building $building */
$building = Building::query()->find($buildingRecord->id);
if (empty($building->flats)) {
continue;
}
// TODO Потом если надо то добавим в базу
$isApartment = false;
$buildingData = [
'building_building_id' => $building->id,
'building_name' => $building->name,
'building_alias' => $building->alias,
'building_street_name' => $building->street_name,
'building_floors' => $building->floors,
'building_region_id' => $building->region_id,
'building_region_district_id' => $building->region_district_id,
'building_metro_id' => $building->metro_id,
'building_city_id' => $building->city_id,
'building_city_district_id' => $building->city_district_id,
'city_district' => $building->cityDistrict?->name,
'metroLineColor' => $building?->subway?->line?->color,
'metroStation' => $building?->subway?->name,
'building_started_year' => $building->started_year,
'building_started_quarter' => $building->started_quarter,
'building_completed_year' => $building->completed_year,
'building_completed_quarter' => $building->completed_quarter,
'building_is_completed' => (bool) $building->is_completed,
'building_developer_id' => $building->developer_id,
'building_latlng' => $building->latlng,
'building_is_special' => (bool) $building->is_special,
'building_special_offer' => $building->special_offer,
'building_queue' => $building->queue,
'building_is_single' => (bool) $building->is_single,
'building_housing_number' => $building->housing_number,
'building_is_own' => $building->is_own,
'building_street_type_id' => $building->street_type_id,
'building_house_number' => $building->house_number,
'building_house_letter' => $building->house_letter,
'building_entrance_count' => $building->entrance_count,
'building_flats_count' => $building->flats_count,
'building_rooms_count' => $building->rooms_count,
'building_finishing_type_id' => $building->finishing_type_id,
'building_house_type_id' => $building->house_type_id,
'building_orientation_id' => $building->orientation_id,
'building_published' => (bool) $building->published,
'building_category_id' => 2,
'building_options' => $buildingRecord->options ?? '',
'building_address' => $building->generateAddress() ?? '',
'building_is_apartment' => (bool) $isApartment,
];
foreach ($building->flats as $flat) {
if (!$flat->is_secondary) {
continue;
}
$flatData = [
'id' => $flat->id,
'area' => $flat->total_area,
'floor' => $flat->floor,
'rooms' => $flat->rooms,
'total_area' => $flat->total_area,
'living_area' => $flat->living_area,
'rooms_area' => $flat->rooms_area,
'kitchen_area' => $flat->kitchen_area,
'number' => $flat->number,
'price' => $flat->price,
'base_price' => $flat->base_price,
'mortage_price' => $flat->mortgage_price ?? 0,
'assigment' => $flat->assignment ?? '',
'contractor' => $flat->contractor,
'bathroom_type_id' => $flat->bathroom_type_id,
'balcony_type_id' => $flat->balcony_type_id,
'finishing_type_id' => $flat->finishing_type_id,
'view_type_id' => $flat->view_type_id,
'entrance' => $flat->entrance,
'rooms_area_separately' => $flat->rooms_area_separately,
'published' => (bool) $flat->published,
'is_secondary' => $flat->is_secondary,
'description' => $flat->description,
'options' => $flat->options->map(fn($option) => ['option_id' => $option->id])->toArray(),
'is_apartment' => (bool) $isApartment,
'gallery' => [],
'gallery_thumb' => [],
'full_thumbs' => [],
'rutube_id' => $flat->rutube_id ?? null,
];
$attachments = ElasticRepository::getAllAttachments($flat);
ElasticRepository::fillGallery($attachments, $flatData);
$indexData = $buildingData + $flatData;
$this->send($this->newIndex, $indexData);
}
}
$this->echo('Добавление вторички завершено');
$this->switchAlias(self::INDEX_SECOND);
$this->clearIndices($this->currentIndices);
}
public function flats(): void
{
if (!$this->prepareIndexing(self::INDEX_FLATS)) {
return;
}
$this->echo('Добавляем данные по квартирам в индекс');
/** @var FlatCollection $flats */
$flats = Flat::query()->where('published', 1)->where('is_secondary', false)->cursor();
$totalCount = $flats->count();
$this->echo('Всего квартир: ' . $totalCount);
foreach ($flats as $flat) {
$building = $flat->building;
if (empty($building) || !$building->published || empty($building->complex) || !$building->complex->published) {
continue;
}
$isApartment = (boolean) $building->complex->is_apartment;
$flatData = [
'id' => $flat->id,
'floor' => $flat->floor,
'area' => $flat->total_area,
'rooms' => $flat->rooms,
'total_area' => $flat->total_area,
'living_area' => $flat->living_area,
'rooms_area' => $flat->rooms_area,
'kitchen_area' => $flat->kitchen_area,
'number' => $flat->number,
'price' => $flat->price,
'base_price' => $flat->base_price,
'mortage_price' => $flat->mortgage_price ?? 0,
'assigment' => $flat->assigment ?? '',
'contractor' => $flat->contractor ?? '',
'bathroom_type_id' => $flat->bathroom_type_id,
'balcony_type_id' => $flat->balcony_type_id,
'finishing_type_id' => $flat->finishing_type_id,
'view_type_id' => $flat->view_type_id,
'entrance' => $flat->entrance,
'rooms_area_separately' => $flat->rooms_area_separately,
'published' => (bool) $flat->published,
'description' => $flat->description,
'options' => $flat->options->map(fn(FlatOption $option) => ['option_id' => $option->id, 'option_name' => $option->name, 'option_alias' => $option->alias])->toArray(),
'is_apartment' => $isApartment,
'building_building_id' => $building->id,
'building_name' => $building->name,
'building_alias' => $building->alias,
'building_street_name' => $building->street_name,
'building_floors' => $building->floors,
'building_region_id' => $building->region_id,
'building_region_district_id' => $building->region_district_id,
'building_metro_id' => $building->metro_id,
'building_city_id' => $building->city_id,
'building_city_district_id' => $building->city_district_id,
'building_city_district' => $building->cityDistrict?->name,
'building_started_year' => $building->started_year,
'building_started_quarter' => $building->started_quarter,
'building_completed_year' => $building->completed_year,
'building_completed_quarter' => $building->completed_quarter,
'building_is_completed' => (bool) $building->is_completed,
'building_developer_id' => $building->developer_id,
'building_latlng' => $building->latlng,
'building_is_special' => (bool) $building->is_special,
'building_special_offer' => $building->special_offer,
'building_queue' => $building->queue,
'building_is_single' => (bool) $building->is_single,
'building_housing_number' => $building->housing_number,
'building_is_own' => $building->is_own,
'building_street_type_id' => $building->street_type_id,
'building_house_number' => $building->house_number,
'building_house_letter' => $building->house_letter,
'building_entrance_count' => $building->entrance_count,
'building_flats_count' => $building->flats_count,
'building_rooms_count' => $building->rooms_count,
'building_finishing_type_id' => $building->finishing_type_id,
'building_house_type_id' => $building->house_type_id,
'building_orientation_id' => $building->orientation_id,
'building_published' => (bool) $building->published,
'building_category_id' => $building->category_id,
'building_address' => $building->generateAddress() ?? '',
'building_is_apartment' => $isApartment,
'gallery' => [],
'gallery_thumb' => [],
'full_thumbs' => [],
'rutube_id' => $flat->rutube_id ?? null,
'complex_id' => (int) $building->complex_id,
'complex_name' => $building->complex->name,
'plan' => null,
'plan_thumb' => null,
];
$attachments = ElasticRepository::getAllAttachments($flat);
ElasticRepository::fillGallery($attachments, $flatData, $flat->plan_id);
$this->send($this->newIndex, $flatData);
}
$this->echo('Добавление квартир завершено');
$this->switchAlias(self::INDEX_FLATS);
$this->clearIndices($this->currentIndices);
}
}