File: /var/www/html/laravel/app/Models/Mel.php
<?php
namespace App\Models;
use Illuminate\Database\Eloquent\Model;
use JsonSchema\Validator;
use GuzzleHttp\Psr7\MultipartStream;
use GuzzleHttp\Client;
/**
* @property $id integer
* @property $content string
* @property $response string
* @property $chatgpt_response string
* @property $prompt_tokens string
* @property $completion_tokens integer
* @property $total_tokens integer
* @property $correct_response string
* @property $comments string
* @property $prompt_id integer
* @property $batch_id integer
* @property $user_id integer
* @property $request_hash string
* @property $match_percent int
* @property $fine_tuning int
*/
class Mel extends Model
{
protected $table = 'mels';
protected $fillable = [
'content',
'response',
'page_number',
'file_name',
'chatgpt_response',
'prompt_tokens',
'completion_tokens',
'total_tokens',
'correct_response',
'comments',
'prompt_id',
'batch_id',
'user_id',
'request_hash',
'match_percent',
'fine_tuning',
'user_id'
];
public static function getValidResponse($hash)
{
$mel = self::query()->where('request_hash', $hash)->whereNotNull('correct_response')->first();
return $mel->correct_response ?? null;
}
public static function hash($content): string
{
$text = $content;
return hash('sha256', $text);
}
public static function sortJson(&$array): void
{
foreach ($array as &$value) {
if (is_array($value)) {
self::sortJson($value);
}
}
ksort($array);
}
public function prepareJsons(): void
{
$this->correct_response = self::prepareJson($this->correct_response);
$this->response = self::prepareJson($this->response);
}
public static function prepareJson($jsonString): string
{
$jsonArray = json_decode($jsonString, true);
if ($jsonArray) {
//self::sortJson($jsonArray);
return json_encode($jsonArray, JSON_UNESCAPED_UNICODE);
}
return $jsonString ? str_replace(array("\r\n", "\r", "\n"), '', $jsonString) : '';
}
public function getText(): string
{
$content = preg_replace("/Q\)(.*)\r\n/", '', $this->content);
return $content;
}
public function reachOutput($correctedResponse = false): string|false
{
if ($correctedResponse == false) {
$correctedResponse = $this->response;
}
$response = json_decode(self::prepareJson($correctedResponse));
return json_encode($response, JSON_UNESCAPED_UNICODE);
}
public function validate(): Mel
{
$this->prepareJsons();
$matchPercent = 100;
if (!$this->correct_response && $this->response) {
$this->correct_response = $this->response;
}
if ($this->correct_response && $this->response) {
$cleanedCorrectResponse = $this->cleanJson($this->prepareJson($this->reachOutput($this->correct_response)));
$cleanedGptResponse = $this->cleanJson($this->prepareJson($this->reachOutput()));
if ($cleanedCorrectResponse !== $cleanedGptResponse) {
similar_text(
$cleanedCorrectResponse,
$cleanedGptResponse,
$matchPercent,
);
}
$this->match_percent = floor($matchPercent);
$validationErrors = $this->validateDataFormat($this->correct_response);
$this->fine_tuning = 0;
if (empty($validationErrors)) {
$this->fine_tuning = 1;
}
Mel::query()->where('id', $this->id)
->update([
'match_percent' => floor($matchPercent),
'fine_tuning' => $this->fine_tuning,
]);
}
return $this;
}
public function cleanJson($jsonString): string
{
$data = json_decode($jsonString, true);
if ($data) {
$data = $this->cleanOptionalAttributes($data);
}
return json_encode($data, JSON_UNESCAPED_UNICODE);
}
public function cleanOptionalAttributes($data): array
{
$optionalAttributes = ['Reference', 'Other', 'CheckNeeded'];
if (is_array($data)) {
foreach ($data as $key => $value) {
if (is_array($value)) {
$data[$key] = $this->cleanOptionalAttributes($value);
} elseif (in_array($key, $optionalAttributes)) {
unset($data[$key]);
}
}
}
return $data;
}
public function minifyJson($jsonString)
{
$data = json_decode($jsonString, true);
return json_encode($data, JSON_UNESCAPED_UNICODE);
}
public function validateDataFormat($json = '')
{
if (is_string($json)) {
$json = json_decode($json);
}
$validator = new Validator();
$schema = str_replace('const', 'pattern', file_get_contents(base_path('public/mel-schema.json')));
$schema = json_decode($schema);
$validator->validate($json, (object) $schema);
$errors = [];
if (!$validator->isValid()) {
foreach ($validator->getErrors() as $error) {
if (!empty($error['constraint']['params'])) {
foreach ($error['constraint']['params'] as $key => $value) {
if (is_array($value)) {
$value = implode(', ', $value);
}
$error['message'] = str_ireplace($value, ":$key", $error['message']);
$error['constraint']['params'][$key] = __($value, [], 'ru');
}
$errors[] = $error['property'] . ": " . __(
$error['message'],
$error['constraint']['params'], 'ru'
);
} else {
$errors[] = $error['property'] . ": " . __($error['message'], [], 'ru');
}
}
}
$errors = $this->compareDates($json, $errors);
return $errors;
}
public function compareDates($json, $errors): array
{
if (!is_array($json) && !is_object($json)) {
return $errors;
}
foreach ($json as $k => $value) {
if (is_object($value) || is_array($value)) {
$errors = $this->compareDates($value, $errors);
} else {
if ($k === 'startDate') {
$startDate = $json->startDate;
if (property_exists($json, 'startTime') && gettype($json->startTime) !== "undefined") {
$startDate .= ' ' . $json->startTime[0] . $json->startTime[1] . ':' . ($json->startTime[2] ?? 0) . ($json->startTime[3] ?? 0);
}
$endDate = null;
if (property_exists($json, 'endDate') && gettype($json->endDate) !== "undefined" && $json->endDate !== 'PERM') {
$endDate = $json->endDate;
if (property_exists($json, 'endTime') && gettype($json->endTime) !== "undefined") {
$endDate .= ' ' . $json->endTime[0] . $json->endTime[1] . ':' . ($json->endTime[2] ?? 0) . ($json->endTime[3] ?? 0);
}
} else {
if ($json->endDate !== 'PERM') {
$errors[] = 'Отсутствует дата окончания';
continue;
}
}
if ($startDate && $endDate) {
if (strtotime($startDate) > strtotime($endDate)) {
$errors[] = 'Время начала действия больше времени окончания: ' . $startDate . ' > ' . $endDate;
}
}
} elseif ($k === 'B_Line') {
$startDate = $this->textToDate($json->B_Line);
$endDate = null;
if (property_exists($json, 'C_Line') && gettype($json->C_Line) !== 'undefined' && $json->C_Line !== 'PERM') {
$endDate = $this->textToDate($json->C_Line);
} elseif ($json->C_Line !== 'PERM') {
$errors[] = 'Отсутствует дата окончания (C_Line)';
continue;
}
if ($startDate && $endDate) {
if (strtotime($startDate) > strtotime($endDate)) {
$errors[] = 'Время начала действия NOTAM больше времени окончания: ' . $startDate . ' > ' . $endDate;
}
}
}
}
}
return $errors;
}
public function textToDate($t)
{
$date = $t;
if (strlen($t) === 10) {
$date = '20' . $t[0] . $t[1] . '-' . $t[2] . $t[3] . '-' . $t[4] . $t[5] . ' ' . $t[6] . $t[7] . ":" . $t[8] . $t[9];
}
return $date;
}
public function process(): void
{
echo "Processing MEL FILE #{$this->id}\n";
if (!$this->content) {
return;
}
$prompt = Prompt::query()
->where('is_active', 1)
->where('type', 'mel')
->first();
$promptText = $prompt->prompt . "\n" . $prompt->json;
$data = [
'input' => $this->getText(),
'key' => env('OPEN_AI_KEY'),
'prompt' => $promptText,
'model' => $prompt->model,
'temperature' => (float) $prompt->temperature,
'max_tokens' => (float) $prompt->max_tokens,
];
print_r($data);
$response = $this->callGptProxi($data);
if (!property_exists($response, 'usage')) {
echo json_encode($response); die;
}
$usage = (array)$response->usage;
$message = $response->choices[0]->message->content ?? '';
$messageData = json_decode($message);
if (is_object($messageData)) {
if (!property_exists($messageData, 'CheckNeeded') || !$messageData->CheckNeeded) {
$messageData->CheckNeeded = true;
$message = json_encode($messageData);
}
} else {
echo $this->comments = "Not valid response from OpenAI";
}
$this->response = $message;
$this->correct_response = $message;
$this->chatgpt_response = json_encode($response);
$this->prompt_tokens = $usage['prompt_tokens'];
$this->completion_tokens = $usage['completion_tokens'];
$this->total_tokens = $usage['total_tokens'];
$this->saveQuietly();
}
public static function getCounters()
{
$prompt = Prompt::query()->where('type', 'mel')->where('is_active', 1)->latest()->first();
$fineTuningJobs = FineTuningJob::query()->where('prompt_id', $prompt->id)->orderBy('created_at', 'desc')->get();
$lastJobDate = date('Y-m-d H:i:s');
if ($fineTuningJobs->first()) {
$lastJobDate = $fineTuningJobs->first()->created_at;
}
$data = [
'fineTuning' => Mel::query()->where('mels.updated_at', '>=', $lastJobDate)
->where('fine_tuning', 1)
->where('correct_response', 'like', "%\"CheckNeeded\":false%")
->count(),
'noFineTuning' => Mel::query()->where('mels.updated_at', '<', $lastJobDate)
->orWhere('fine_tuning', 0)
->orWhere('correct_response', 'like', "%\"CheckNeeded\":true%")
->count(),
'checkNeeded' => Mel::query()
->where(function ($query) use ($lastJobDate) {
$query->where('correct_response', 'like', "%\"CheckNeeded\":true%")
->orWhere(function ($query2) use ($lastJobDate) {
$query2->whereNull('correct_response')
->where('response', 'like', "%\"CheckNeeded\":true%");
});
})
->count(),
'noCheckNeeded' => Mel::query()
->where(function ($query) use ($lastJobDate) {
$query->where('correct_response', 'like', "%\"CheckNeeded\":false%")
->orWhere(function ($query2) use ($lastJobDate) {
$query2->whereNull('correct_response')
->where('response', 'like', "%\"CheckNeeded\":false%");
});
})->count(),
'usedInFineTuning' => 0,
'modifiedAfterFineTuning' => 0,
];
if ($lastJobDate) {
$data['usedInFineTuning'] = Mel::query()->where('fine_tuning', 1)->where('correct_response', 'like', "%\"CheckNeeded\":false%")->where('updated_at', '<', $lastJobDate)->count();
$data['modifiedAfterFineTuning'] = Mel::query()->where('fine_tuning', 0)->orWhere('correct_response', 'like', "%\"CheckNeeded\":true%")->orWhere('updated_at', '>=', $lastJobDate)->count();
}
return $data;
}
public function callGptProxi($data)
{
$url = 'https://notam.foach.site/gpt.php';
$data = [
'key' => $data['key'],
'input' => $data['input'],
'model' => $data['model'],
'prompt' => $data['prompt'] ?? " ",
'messages' => $data['messages'] ?? [],
'temperature' => (float)$data['temperature'] ?? 0.7,
'max_tokens' => (float)$data['max_tokens'] ?? 4096,
];
$curl = curl_init();
curl_setopt($curl, CURLOPT_RETURNTRANSFER, true);
curl_setopt($curl, CURLOPT_URL, $url);
curl_setopt($curl, CURLOPT_HTTPHEADER, ['Content-Type:application/json']);
curl_setopt($curl, CURLOPT_HEADER, false);
curl_setopt($curl, CURLOPT_CUSTOMREQUEST, 'POST');
curl_setopt($curl, CURLOPT_POSTFIELDS, json_encode($data));
curl_setopt($curl, CURLOPT_SSL_VERIFYPEER, 0);
curl_setopt($curl, CURLOPT_SSL_VERIFYHOST, 0);
$result = curl_exec($curl);
curl_close($curl);
return json_decode($result);
}
}