<?php
use LWVendor\Danny50610\BpeTokeniser\EncodingFactory;
use LWVendor\PhpOffice\PhpSpreadsheet\Calculation\MathTrig\Trig\Cotangent;

/**
 * AI controller
 */
class Wpil_AI
{
    private static $ai;
    private static $ai_service_connected = null;
    public static $question_limit = 6000;
    public static $concurrency = 100;
    public static $magic = '';
    public static $magnatude_cache = array();
    public static $dot_product_cache = array();
    public static $batch_limit = null;
    public static $cached_embedding_data = array();
    public static $cached_post_sentence_embedding_data = array();
    public static $status_cache = array();
    public static $complete_log_cache = array();
    public static $chunked_posts = array();
    public static $query_ids = array();
    public static $sentence_anchor_cache = array();
    public static $origin_post = null;
    public static $purpose = null;
    public static $model = null;
    public static $rate_limited = false;
    public static $insufficient_quota = false;
    public static $user_not_exist = false;
    public static $invalid_request = false;
    public static $invalid_api_key = false;
    public static $error_message = '';
    public static $error_log = array();
    public static $current_error = false;
    public static $anchor_assessment_ids = array();
    public static $active_linking_process_key = '';
    public static $keyword_cannibalization_keywords = null;
    public static $active_credit_tracking_context = array();
    public static $request_credit_tracking_runs = array();

    function __construct()
    {
        // TODO: Add license checks && settings
        require trailingslashit(WP_INTERNAL_LINKING_PLUGIN_DIR) . 'vendor/orhanerday/open-ai/src/OpenAi.php';
        require trailingslashit(WP_INTERNAL_LINKING_PLUGIN_DIR) . 'vendor/orhanerday/open-ai/src/Url.php';
        require trailingslashit(WP_INTERNAL_LINKING_PLUGIN_DIR) . 'vendor/danny50610/bpe-tokeniser/src/Encoding.php';
        require trailingslashit(WP_INTERNAL_LINKING_PLUGIN_DIR) . 'vendor/danny50610/bpe-tokeniser/src/EncodingFactory.php';

        // todo: temp!
        self::$batch_limit = 50000;

        self::$ai_service_connected = (Wpil_Settings::get_linkwhisper_ai_active() && Wpil_Settings::get_linkwhisper_ai_token());

        if(self::$ai_service_connected){
            self::$concurrency = 100;
        }else{
            $key = Wpil_Settings::getOpenAIKey();

            if(empty($key)){
                return;
            }

            if(empty(self::$ai)){
                self::$ai = new \LWVendor\Orhanerday\OpenAi\OpenAi($key);
            }

            self::$ai->setTimeout(Wpil_Base::overTimeLimit(5, null, true));
            self::$ai->setConcurrency(self::$concurrency);
        }
    }

    public function register()
    {
        add_action('wp_ajax_wpil_live_download_ai_data', [__CLASS__, 'ajax_live_download_ai_data']);
        add_action('wp_ajax_wpil_live_ai_linking', [__CLASS__, 'ajax_live_ai_linking']);
        add_action('wp_ajax_wpil_clear_ai_data', [__CLASS__, 'ajax_wpil_clear_ai_data']);
        add_action('wp_ajax_wpil_clear_ai_relation_data', [__CLASS__, 'ajax_wpil_clear_ai_relation_data']);
        add_action('wp_ajax_wpil_clear_ai_keyword_data', [__CLASS__, 'ajax_wpil_clear_ai_keyword_data']);
        add_action('wp_ajax_wpil_clear_ai_linking_process_data', [__CLASS__, 'ajax_clear_ai_linking_process_data']);
        add_action('wp_ajax_wpil_ai_dismiss_credit_notice', [__CLASS__, 'ajax_wpil_dismiss_credit_notice']);
        add_action('wp_ajax_wpil_ai_dismiss_api_key_decoding_error', [__CLASS__, 'ajax_wpil_dismiss_api_key_decoding_error']);
        add_action('wp_ajax_wpil_estimate_site_processing_cost', [__CLASS__, 'ajax_estimate_site_processing_cost']);
        add_action('wp_ajax_setup_user_ai_subscription', array(__CLASS__, 'ajax_setup_user_ai_subscription'));
        add_action('wp_ajax_clear_user_ai_subscription', array(__CLASS__, 'ajax_clear_user_ai_subscription'));
        add_filter('cron_schedules', [__CLASS__, 'add_batch_cron_interval']);
        add_action('admin_init', [__CLASS__, 'schedule_batch_process']);
        add_action('wpil_ai_batch_process_cron', [__CLASS__, 'perform_cron_batch_process']);
        add_filter('orhanerday_openai_stream_response_data', [__CLASS__, 'process_streamed_data'], 10, 3);
        add_action('wp_ajax_wpil_get_review_links', array(__CLASS__, 'ajax_get_review_links'));
        add_action('wp_ajax_wpil_get_review_link_count', array(__CLASS__, 'ajax_get_review_link_count'));
        add_action('wp_ajax_wpil_set_review_link_decision', array(__CLASS__, 'ajax_set_review_link_decision'));
        add_action('wp_ajax_wpil_get_wizard_credit_estimate', [__CLASS__, 'ajax_get_wizard_credit_estimate']);
    }

    /**
     * @return array
     **/
    public static function call_linkwhisper_ai($content = '', $model = '', $action = '', $params = []){
        if(is_array($content)){
            $args = array(
                'user_id' => Wpil_Settings::get_linkwhisper_ai_user_id(),
                'action' => (!empty($action)) ? $action: self::$purpose,
                'message_list' => $content,
                'input' => '',
                'model' => (!empty($model)) ? $model: self::$model,
                'access_token' => Wpil_Settings::get_linkwhisper_ai_token(),
                'url' => site_url()
            );
        }else{
            $args = array(
                'user_id' => Wpil_Settings::get_linkwhisper_ai_user_id(),
                'action' => (!empty($action)) ? $action: self::$purpose,
                'input' => $content,
                'model' => (!empty($model)) ? $model: self::$model,
                'access_token' => Wpil_Settings::get_linkwhisper_ai_token(),
                'url' => site_url()
            );
        }

        if(!empty($params)){
            $args['params'] = $params;
        }

        // exist if there are 
        /*$creds = self::get_available_ai_credits();
        if(empty($creds)){
            return array();
        }*/

        $response = null;
        if(is_array($content)){
            $response = self::sendMultiRequest($args);
        }else{
            $response = self::sendRequest($args);
        }

        return $response; 
    }

    /**
     * @param  string  $url
     * @param  string  $method
     * @param  array   $opts
     * @return bool|string
     */
    private static function sendRequest($opts = [])
    {
        $post_fields = wp_json_encode($opts);
        $headers = array("Content-Type: application/json");
        $curl_info = [
            CURLOPT_USERAGENT      => WPIL_DATA_USER_AGENT,
            CURLOPT_URL            => 'https://api.linkwhisper.com/',
            CURLOPT_RETURNTRANSFER => true,
            CURLOPT_ENCODING       => '',
            CURLOPT_MAXREDIRS      => 10,
            CURLOPT_TIMEOUT        => 60,
            CURLOPT_FOLLOWLOCATION => true,
            CURLOPT_HTTP_VERSION   => CURL_HTTP_VERSION_1_1,
            CURLOPT_POST           => 1,
            CURLOPT_POSTFIELDS     => $post_fields,
            CURLOPT_HTTPHEADER     => $headers
        ];

        if ($opts == []) {
            unset($curl_info[CURLOPT_POSTFIELDS]);
        }

        $curl = curl_init();
        curl_setopt_array($curl, $curl_info);
        $response = curl_exec($curl);
        curl_close($curl);
        return $response;
    }

    /**
     * @param  string  $url
     * @param  string  $method
     * @param  array   $opts
     * @return bool|string
     */
    private static function sendMultiRequest($opts = [])
    {
        // create the multihandle
        $mh = curl_multi_init();
        $handles = array();
        $messages = $opts['message_list'];
        unset($opts['message_list']);

        for($i = 0; $i < self::$concurrency; $i++){
            if(!isset($messages[$i])){
                break;
            }

            $curl_opts = array_merge($opts, ['input' => $messages[$i]]);
            $post_fields    = wp_json_encode($curl_opts);
            $headers = array("Content-Type: application/json");

            $curl_info = [
                CURLOPT_URL            => 'https://api.linkwhisper.com/',
                CURLOPT_RETURNTRANSFER => true,
                CURLOPT_ENCODING       => '',
                CURLOPT_MAXREDIRS      => 10,
                CURLOPT_TIMEOUT        => 60,
                CURLOPT_FOLLOWLOCATION => true,
                CURLOPT_HTTP_VERSION   => CURL_HTTP_VERSION_1_1,
                CURLOPT_POST           => 1,
                CURLOPT_POSTFIELDS     => $post_fields,
                CURLOPT_HTTPHEADER     => $headers,
                CURLOPT_BUFFERSIZE     => 65536,
            ];
            if ($curl_opts == []) {
                unset($curl_info[CURLOPT_POSTFIELDS]);
            }

            $handles[$i] = curl_init();
            curl_setopt_array($handles[$i], $curl_info);
            curl_multi_add_handle($mh, $handles[$i]);
        }

        if(!empty($handles)){
            do {
                $status = curl_multi_exec($mh, $active);
                
                // Check if any handle has completed
                while ($info = curl_multi_info_read($mh)) {
                    $handle = $info['handle'];

                    // Only process if the handle completed successfully
                    if ($info['result'] === CURLE_OK) {
                        // Find the array key associated with this handle
                        $handle_id = array_search($handle, $handles, true);

                        // Get content from the handle
                        $content = curl_multi_getcontent($handle);
                        $info    = curl_getinfo($handle);

                        $processed = apply_filters('orhanerday_openai_stream_response_data', $handle_id, $content, $info);
                        if($processed){
                            // Remove the handle once processed
                            curl_multi_remove_handle($mh, $handle);
                            curl_close($handle);
                            unset($handles[$handle_id]);
                        }
                    }
                }
                
                if ($active) {
                    curl_multi_select($mh);
                }
            } while ($active && $status == CURLM_OK);
        }

        $responses = array();
        foreach($handles as $handle_id => $handle){
            $responses[$handle_id] = curl_multi_getcontent($handle);
            curl_multi_remove_handle($mh, $handle);
            curl_close($handle);
        }
        curl_multi_close($mh);
        return $responses;
    }

    /**
     * Does stuff...
     * Ok fine, it does live ai linking.
     * For the One Click Setup
     **/
    public static function ajax_live_ai_linking(){
        Wpil_Base::verify_nonce('wpil_download_ai_data');
        // be sure to ignore any external object caches
        Wpil_Base::ignore_external_object_cache();
        // Remove any hooks that may interfere with AJAX requests
        Wpil_Base::remove_problem_hooks();
        $ai_linking_enabled = self::is_ai_linking_enabled_request();

        // makre sure that we don't call too many times
        $checkout = get_transient('wpil_ai_live_linking_waiter');
        if(!empty($checkout) && $checkout > (time() - 1)){
            wp_send_json_error(['message' => 'Already running process!'], 400);
        }
        set_transient('wpil_ai_live_linking_waiter', time(), 1); // we're just preventing doubling up, so it is a short lived transient

        // get the current process key
        $process_key = get_option('wpil_ai_linking_process_key', '');
        $queue_reset = !empty($_POST['queue_reset']);

        // set the key if there isn't one
        if(empty($process_key)){
            $process_key = md5('one-click-setup');//md5(time()); // todo: possibly go back to time if needed
            $queue_reset = true;
        }

        if($queue_reset){
            self::reset_ai_linking_process_runtime($process_key, false);
            Wpil_LinkMapping::reset_relation_map_queue($process_key);
            self::begin_credit_tracking_task_run(
                self::get_credit_tracking_task_key_from_linking_process_key($process_key),
                'linking:' . $process_key,
                true
            );
            update_option('wpil_ai_linking_process_key', $process_key, false);
        }elseif(get_option('wpil_ai_linking_process_key', '') !== $process_key){
            update_option('wpil_ai_linking_process_key', $process_key, false);
        }

        if(Wpil_LinkMapping::get_relation_map_total_item_count($process_key) < 1){
            Wpil_LinkMapping::reset_relation_map_queue($process_key);
        }

        $map_processed = Wpil_LinkMapping::is_map_processed($process_key);
        $available_count = Wpil_LinkMapping::get_relation_map_pending_ai_item_count($process_key, array(), array(), true);
        if(!$map_processed && $available_count <= 10){
            // fire off the relational mapping
            $results = Wpil_LinkMapping::build_relationship_map($process_key);

            $map_processed = Wpil_LinkMapping::is_map_processed($process_key);
            $available_count = Wpil_LinkMapping::get_relation_map_pending_ai_item_count($process_key, array(), array(), true);
            // if the mapps not finished and still produced no actionable rows, go around for another try
            if(!$results && !$map_processed && $available_count < 1){
                wp_send_json(array('continue' => array(
                    'progress' => 1,
                    'review_ready_count' => self::get_process_review_link_count($process_key),
                )));
            }
        }

        $direction = 'outbound'; // change to outbound if the user oks us linking all over the place
        $review_ready_count = self::get_process_review_link_count($process_key);

        $pending_count = Wpil_LinkMapping::get_relation_map_pending_ai_item_count($process_key);
        $map_total = max(1, Wpil_LinkMapping::get_relation_map_total_item_count($process_key));
        $ai_processed_count = Wpil_LinkMapping::get_relation_map_ai_processed_item_count($process_key);

        if(!$map_processed && $available_count <= 10){
            $map_progress = min(99, (int) round((Wpil_LinkMapping::get_relation_map_completed_item_count($process_key) / $map_total) * 100));
            wp_send_json(array('continue' => array(
                'progress' => max(1, $map_progress),
                'ai_credits' => self::get_available_ai_credits(),
                'estimated_credit_cost' => self::estimate_site_processing_credit_cost($process_key, $ai_linking_enabled),
                'review_ready_count' => $review_ready_count,
            )));
        }

        // get the next batch of mapped posts to use for AI linking
        $pending_rows = Wpil_LinkMapping::get_relation_map_items_pending_ai($process_key, 11);

        // find the first available post with relations
        $post_pid = '';
        foreach($pending_rows as $row){
            if(empty($row->pid)){
                continue;
            }

            $pid = self::normalize_pid($row->pid);
            $m = !empty($row->map_data) && is_array($row->map_data) ? $row->map_data : array();

            if(empty($m)){
                Wpil_LinkMapping::mark_relation_map_ai_processed($process_key, $pid, true);
                continue;
            }

            if(isset($m['pillar_content']) && !empty($m['pillar_content'])){
                $direction = 'inbound'; // make sure pillar content always gets inbound links
            }

            $post_pid = $pid;
            break;
        }

        // if there isn't one
        if(empty($post_pid)){
            $pending_count = Wpil_LinkMapping::get_relation_map_pending_ai_item_count($process_key);
            if($map_processed && $pending_count < 1){
                // we're done!
                wp_send_json(array('success' => array(
                    'progress' => 100,
                    'ai_credits' => self::get_available_ai_credits(),
                    'estimated_credit_cost' => 0,
                    'review_ready_count' => $review_ready_count,
                )));
            }

            $percent = round(($ai_processed_count / $map_total) * 100, 2);
            wp_send_json(array('continue' => array(
                'progress' => $percent,
                'ai_credits' => self::get_available_ai_credits(),
                'estimated_credit_cost' => self::estimate_site_processing_credit_cost($process_key, $ai_linking_enabled),
                'review_ready_count' => $review_ready_count,
            )));
        }

        Wpil_LinkMapping::claim_relation_map_ai_item($process_key, $post_pid);

        // chew those with AI first
        self::create_link_suggestions_from_map($post_pid, $process_key, $direction);

        // if the user wants to review, let him see the results
        // if not, go ahead and insert them.
        $auto_insert = (isset($_POST['approve_all']) && !empty($_POST['approve_all'])) || (isset($_POST['link_mode']) && !empty($_POST['link_mode']) && $_POST['link_mode'] === 'auto');
        if($auto_insert){
            self::auto_create_links_from_suggestions(0, '', $process_key);
        }

        Wpil_LinkMapping::mark_relation_map_ai_processed($process_key, $post_pid, true);
        $available_count = Wpil_LinkMapping::get_relation_map_pending_ai_item_count($process_key, array(), array(), true);
        $pending_count = Wpil_LinkMapping::get_relation_map_pending_ai_item_count($process_key);
        $ai_processed_count = Wpil_LinkMapping::get_relation_map_ai_processed_item_count($process_key);
        $percent = round(($ai_processed_count / $map_total) * 100, 2);
        $review_ready_count = self::get_process_review_link_count($process_key);

        if($map_processed && $pending_count < 1){
            delete_option('wpil_ai_linking_process_key'); // delete the process key when we're done
            self::clear_credit_tracking_task_run('linking:' . $process_key);
            wp_send_json(array('success' => array(
                'progress' => 100,
                'ai_credits' => self::get_available_ai_credits(),
                'estimated_credit_cost' => 0,
                'review_ready_count' => $review_ready_count,
            )));
        }else{
            wp_send_json(array('continue' => array(
                'progress' => $percent,
                'ai_credits' => self::get_available_ai_credits(),
                'estimated_credit_cost' => self::estimate_site_processing_credit_cost($process_key, $ai_linking_enabled),
                'review_ready_count' => $review_ready_count,
            )));
        }
    }

    private static function get_process_review_link_count($process_key = ''){
        global $wpdb;

        $process_key = is_string($process_key) ? sanitize_text_field($process_key) : '';
        if(empty($process_key)){
            return 0;
        }

        $table = $wpdb->prefix . 'wpil_ai_linking';
        $sql = $wpdb->prepare(
            "SELECT COUNT(*)
                FROM {$table}
                WHERE inserted = 0
                    AND ignored = 0
                    AND process_key = %s",
            $process_key
        );

        return (int) $wpdb->get_var($sql);
    }

    /**
     * Gets the ids that have been processeed
     **/
    public static function get_linking_suggestion_processed_ids($process_key = '', $direction = 'outbound'){
        global $wpdb;
        $linking_table = $wpdb->prefix . 'wpil_ai_linking';

        $search = "CONCAT(`post_type`, '_', `post_id`)";
        if($direction === 'inbound'){
            $search = "CONCAT(`target_type`, '_', `target_id`)";
        }

        $sql = "SELECT DISTINCT {$search} FROM {$linking_table} WHERE 1=1";
        if(!empty($process_key)){
            $sql .= $wpdb->prepare(" AND process_key = %s", $process_key);
        }

        $ids = $wpdb->get_col($sql);

        return (!empty($ids)) ? $ids: [];
    }

    /**
     * 
     **/
    public static function ajax_live_download_ai_data(){
        Wpil_Base::verify_nonce('wpil_download_ai_data');
        // be sure to ignore any external object caches
        Wpil_Base::ignore_external_object_cache();
        // Remove any hooks that may interfere with AJAX requests
        Wpil_Base::remove_problem_hooks();
        $ai_linking_enabled = self::is_ai_linking_enabled_request();

        $selected_processes = Wpil_Settings::get_selected_ai_batch_processes(true);
        $processed_embeddings = false;
        $initial_stats = self::get_completed_post_stats(true);
        $total_posts = self::get_total_processable_posts();
        $post_saving = true;
        $last_pass_unchanged = (array_key_exists('last_pass_unchanged', $_POST) && $_POST['last_pass_unchanged'] === '1') ? true: false;

        // set a flag so that we know that we're downloading data
        set_transient('wpil_doing_ai_data_download', time(), MINUTE_IN_SECONDS * 3);

        // if the batch processing is supposed to be turned on
        if(isset($_POST['activate_batch_processing']) && !empty($_POST['activate_batch_processing'])){
            // turn it on
            update_option('wpil_enable_ai_batch_processing', '1');
        }

        $start_time = isset($_POST['start_time']) && !empty($_POST['start_time']) ? (int)$_POST['start_time']: time();
        $current_process = esc_html__('Sending Site Data to OpenAI be Processed', 'wpil');

        // if the batch processing is supposed to be turned on
        if(isset($_POST['activate_batch_processing']) && !empty($_POST['activate_batch_processing'])){
            // turn it on
            update_option('wpil_enable_ai_batch_processing', '1');
        }

        // if this is the first go round
        if(isset($_POST['start_time']) && empty($_POST['start_time'])){
            // and we're connected to the ai service
            if(self::$ai_service_connected){
                // do a credit check
                $credit = self::get_available_ai_credits(true);
                if($credit < 1){
                    self::$insufficient_quota = true;
                }
            }

            // clear the embedding id lock
            self::set_last_embedding_id_lock();
        }

        if(in_array('create-post-embeddings', $selected_processes)){
            $current_process = esc_html__('Generating AI Relation Data...', 'wpil');

            // if we're not running the ai service
            if(!self::$ai_service_connected){
                // up the number of concurrant processes we'll run for the post embeddings
                self::$concurrency = Wpil_Settings::get_ai_process_limit('create-post-embeddings', true);
                self::$ai->setConcurrency(self::$concurrency);
            }

            self::create_site_embeddings();

            // if we're still not running the ai service
            if(!self::$ai_service_connected){
                // set the concurrancy back to where it should be
                self::$concurrency = 100;
                self::$ai->setConcurrency(self::$concurrency);
            }
        }

        // if there are no embeddings currently being processed and we still have time
        $has_completed_embeddings = self::has_completed_post_embedding_calculations();
        if(!Wpil_Base::overTimeLimit(5, 35) && self::has_completed_post_embeddings() && !$has_completed_embeddings){
            $current_process = esc_html__('Calculating AI Relation Scores...', 'wpil');
            $processed_embeddings = self::stepped_calculate_post_embeddings();
            if($processed_embeddings){
                $total = count(self::get_calculated_embedding_post_ids());
                $current_process .= sprintf(esc_html__(' %d Total Posts Scored...', 'wpil'), $total);
            }

            // clear the old AI Sitemap
            Wpil_Sitemap::delete_sitemap(false, 'ai_sitemap');
        }

        if($has_completed_embeddings && !Wpil_Sitemap::has_sitemap('ai_sitemap')){
            $relatedness = Wpil_AI::calculate_relatedness_sitemap();
            Wpil_Sitemap::save_sitemap($relatedness, 'ai_sitemap', 'AI Sitemap');
        }

        // if we haven't made downloading progress and there's time
        if($last_pass_unchanged && !Wpil_Base::overTimeLimit(5, 20)){
            // try to process any available keywords
            $post_saving = self::do_post_save_finishing();
            if(!$post_saving){
                $current_process = esc_html__('Processing Site Data...', 'wpil');
            }
        }

        if(!Wpil_Base::overTimeLimit(5, 20)){
            $current_process = esc_html__('Analyzing Site Posts...', 'wpil');
            self::analyze_site_posts();
        }

        if(!Wpil_Base::overTimeLimit(5, 20)){
            $post_saving = self::do_post_save_finishing();
            if(!$post_saving){
                $current_process = esc_html__('Processing Site Data...', 'wpil');
            }

            if(!Wpil_Sitemap::has_sitemap('ai_product_sitemap') && self::check_batch_status_completed(3, true)){
                $products = Wpil_AI::calculate_product_sitemap();
                if(!empty($products)){
                    Wpil_Sitemap::save_sitemap($products, 'ai_product_sitemap', 'AI-Detected Product Sitemap');
                }
            }
        }

        $current_stats = self::get_completed_post_stats(true, true);
        $all_processed = array();
        $completed = false;
        $live_processed_results = array();
        $oai_completed = array();

        if(!empty($current_stats)){
            foreach($current_stats as $ind => $count){
                if((int)$count >= (int)$total_posts){
                    $all_processed[$ind] = true;

                    if(in_array($ind, $selected_processes)){
                        $oai_completed[$ind] = true;
                    }
                }

                if(array_key_exists($ind, $initial_stats)){
                    $live_processed_results[$ind] = ($count - $initial_stats[$ind]);
                }else{
                    $live_processed_results[$ind] = $count;
                }

                if(empty($live_processed_results[$ind])){
                    unset($live_processed_results[$ind]);
                }
            }

            if(count(array_filter($all_processed)) === count($current_stats)){
                $completed = true;
            }
        }

        $oai_completed = (count(array_filter($oai_completed)) === count($selected_processes)) ? true: false;

        $response = array();
        if(self::$insufficient_quota || self::$invalid_request || self::$invalid_api_key || self::$user_not_exist){
            if(self::$insufficient_quota){
                update_option('wpil_oai_insufficient_quota_error', '1');
            }
            $response = (self::$ai_service_connected) ? array('error' => self::get_linkwhisper_ai_error_message()): array('error' => self::get_live_oai_error_message());
        }elseif(!$completed || !$post_saving || $processed_embeddings > 0){
            $response = array(
                'continue' => array(
                    'data' => $live_processed_results,
                    'data_total_processed' => $current_stats,
                    'oai_completed' => $oai_completed,
                    'all' =>  $all_processed,
                    'post_saving' => $post_saving,
                    'processed_embeddings' => $processed_embeddings,
                    'estimated_cost' => self::calculate_token_cost_by_time($start_time),
                    'current_process' => $current_process,
                    'start_time' => $start_time,
                    'completed' => $completed,
                    'lock' => self::get_last_embedding_id_lock(),
                    'completion_messages' => array(
                        'info' => array(
                            'title' => __('Processing Halted', 'wpil'), 
                            'text' => __('Link Whisper is not currently able to process any more posts. The reason for this is unclear, there may have been an error, or it could be because all the posts are finished processing. Please check the System Error Log to see if there are any errors, and the Content Processing Status to see if all of the posts are processed.', 'wpil')
                        ),
                        'error' => (self::$ai_service_connected) ? self::get_linkwhisper_ai_error_message() : self::get_live_oai_error_message()
                        ),
                    'is_rate_limited' => self::$rate_limited,
                    'ai_credits' => self::get_available_ai_credits(),
                    'estimated_credit_cost' => self::estimate_site_processing_credit_cost(get_option('wpil_ai_linking_process_key', null), $ai_linking_enabled)
                )
            );
        }else{
            $response = array(
                'success' => array(
                    'title' => __('Processing Complete!', 'wpil'),
                    'text'  => __('All available site data has been processed!', 'wpil'),
                    'oai_completed' => $oai_completed,
                    'ai_credits' => self::get_available_ai_credits(),
                    'estimated_credit_cost' => self::estimate_site_processing_credit_cost(get_option('wpil_ai_linking_process_key', null), $ai_linking_enabled)
                )
            );
        }

        wp_send_json($response);
    }

    public static function ajax_wpil_clear_ai_data(){
        Wpil_Base::verify_nonce('wpil_clear_ai_data');

        $cleared =  self::clear_ai_data();

        $response = array();
        if($cleared){
            $response = array(
                'success' => array(
                    'title' => __('Data Cleared!', 'wpil'),
                    'text'  => __('All AI generated data has been deleted.', 'wpil'),
                )
            );
        }else{
            $response = array(
                'error' => array(
                    'title' => __('Unknown Error', 'wpil'),
                    'text'  => __('Unfortunately, there was an error while trying to clear the AI data, and there may still be some stored on the site.', 'wpil'),
                )
             );
        }

        wp_send_json($response);
    }

    public static function ajax_wpil_clear_ai_relation_data(){
        Wpil_Base::verify_nonce('wpil_clear_ai_data');

        $cleared =  self::clear_ai_data(['relation_analysis']);

        $response = array();
        if($cleared){
            $response = array(
                'success' => array(
                    'title' => __('Data Cleared!', 'wpil'),
                    'text'  => __('All AI Relation Analysis data has been deleted.', 'wpil'),
                )
            );
        }else{
            $response = array(
                'error' => array(
                    'title' => __('Unknown Error', 'wpil'),
                    'text'  => __('Unfortunately, there was an error while trying to clear the AI Relation Analysis data, and there may still be some stored on the site.', 'wpil'),
                )
             );
        }

        wp_send_json($response);
    }

    public static function ajax_wpil_clear_ai_keyword_data(){
        Wpil_Base::verify_nonce('wpil_clear_ai_data');

        $cleared =  self::clear_ai_data(['keyword_analysis']);
        // also remove any assigned ai generated target keywords from the target keyword report!
        Wpil_TargetKeyword::delete_keyword_by_type('ai-generated-keyword');

        $response = array();
        if($cleared){
            $response = array(
                'success' => array(
                    'title' => __('Data Cleared!', 'wpil'),
                    'text'  => __('All AI Keywords have been deleted.', 'wpil'),
                )
            );
        }else{
            $response = array(
                'error' => array(
                    'title' => __('Unknown Error', 'wpil'),
                    'text'  => __('Unfortunately, there was an error while trying to clear the AI Keywords, and there may still be some stored on the site.', 'wpil'),
                )
             );
        }

        wp_send_json($response);
    }

    public static function ajax_clear_ai_linking_process_data(){
        Wpil_Base::verify_nonce('wpil_clear_ai_linking_process_data');

        if(!current_user_can('manage_options')){
            wp_send_json(array(
                'error' => array(
                    'title' => __('Permission Error', 'wpil'),
                    'text'  => __('You do not have permission to perform this action.', 'wpil'),
                )
            ));
        }

        $cleared = self::clear_ai_linking_process_data();

        if($cleared){
            wp_send_json(array(
                'success' => array(
                    'title' => __('Data Cleared!', 'wpil'),
                    'text'  => __('All AI Linking process data has been deleted.', 'wpil'),
                )
            ));
        }

        wp_send_json(array(
            'error' => array(
                'title' => __('Unknown Error', 'wpil'),
                'text'  => __('Unfortunately, there was an error while trying to clear the AI Linking process data.', 'wpil'),
            )
        ));
    }

    /**
     * Clears only AI linking process artifacts used by AI linking/fix runs.
     * Does not clear broader AI datasets or unrelated site data.
     **/
    public static function clear_ai_linking_process_data(){
        global $wpdb;

        $all_ok = true;
        $ai_linking_table = $wpdb->prefix . 'wpil_ai_linking';
        $relation_map_table = $wpdb->prefix . 'wpil_relation_mapping';

        $tables = array($ai_linking_table, $relation_map_table);
        foreach($tables as $table){
            $table_exists = $wpdb->get_var($wpdb->prepare("SHOW TABLES LIKE %s", $table));
            if($table_exists === $table){
                $result = $wpdb->query("TRUNCATE TABLE {$table}");
                if($result === false){
                    $all_ok = false;
                }
            }
        }

        // Clear active fix tracker and registry for dashboard Fix with AI.
        delete_option('wpil_ai_fix_registry');
        delete_transient('wpil_doing_ai_fix_process');
        $linking_process_keys = array(
            md5('one-click-setup'),
            md5('orphan-post-search'),
            md5('link-coverage-search'),
            md5('link-quality-search'),
            md5('broken-link-search'),
            md5('external-focus-search'),
        );
        foreach($linking_process_keys as $process_key){
            self::clear_credit_tracking_task_run('linking:' . $process_key);
        }

        // Remove AI linking process options (process key + processed-id trackers).
        $option_names = $wpdb->get_col(
            "SELECT option_name
                FROM {$wpdb->options}
                WHERE option_name LIKE 'wpil_ai_linking_%'
                    OR option_name LIKE 'wpil_link_map_snapshot_%'"
        );
        if(!empty($option_names)){
            foreach($option_names as $option_name){
                if(!is_string($option_name) || $option_name === ''){
                    continue;
                }
                delete_option($option_name);
            }
        }

        // Remove review/fix transient rows for all users/process keys.
        $transient_cleanup = $wpdb->query(
            "DELETE FROM {$wpdb->options}
                WHERE option_name LIKE '_transient_wpil_review_served_%'
                    OR option_name LIKE '_transient_timeout_wpil_review_served_%'
                    OR option_name LIKE '_transient_wpil_map_replace_links_id_list_%'
                    OR option_name LIKE '_transient_timeout_wpil_map_replace_links_id_list_%'"
        );
        if($transient_cleanup === false){
            $all_ok = false;
        }

        // Clear current user's in-memory review list immediately as well.
        delete_transient('wpil_review_served_' . get_current_user_id());

        return $all_ok;
    }

    /**
     * Clears the saved queue trail for a single AI linking run so the next pass starts fresh.
     **/
    private static function reset_ai_linking_process_runtime($process_key = '', $clear_process_key = true){
        global $wpdb;

        $process_key = is_string($process_key) ? sanitize_text_field($process_key) : '';
        if(empty($process_key)){
            return;
        }

        //$linking_table = $wpdb->prefix . 'wpil_ai_linking'; // TODO: Think about and possibly reenable later
        //$wpdb->delete($linking_table, array('process_key' => $process_key), array('%s'));

        if($clear_process_key && get_option('wpil_ai_linking_process_key', '') === $process_key){
            delete_option('wpil_ai_linking_process_key');
        }

        delete_option('wpil_ai_linking_' . $process_key);
        delete_transient('wpil_review_served_' . get_current_user_id());

        Wpil_LinkMapping::delete_relation_map($process_key);
        self::clear_credit_tracking_task_run('linking:' . $process_key);
    }

    public static function ajax_wpil_dismiss_credit_notice(){
        update_option('wpil_oai_insufficient_quota_error', '0');
    }

    public static function ajax_wpil_dismiss_api_key_decoding_error(){
        update_option('wpil_open_ai_key_decoding_error', '0');
        update_option('wpil_ai_token_decoding_error', '0'); // also update the Link Whisper AI since the user _should_ have updated the wp encryption tokens needed to run the system
    }

    public static function ajax_estimate_site_processing_cost(){
        // create a list of all the posts that need to be processed
        // create a counter that will keep track of their estimated processing costs
        // create something that will loop over each one and estimate based onteh content
        // return the result to the customer

        if(isset($_POST['reset']) && !empty($_POST['reset'])){
            delete_transient('wpil_ai_content_estimate_post_ids');
            delete_transient('wpil_ai_content_estimate_post_id_count');
            delete_transient('wpil_ai_content_estimate_cost');
        }

        $mode = (isset($_POST['estimate_mode']) && $_POST['estimate_mode'] === 'link-whisper') ? 'link-whisper': 'direct';

        $cost = get_transient('wpil_ai_content_estimate_cost');
        if(empty($cost)){
            $cost = 0;
        }

        $processable = get_transient('wpil_ai_content_estimate_post_ids');
        $total = get_transient('wpil_ai_content_estimate_post_id_count');
        if(empty($processable)){
            // for the time being, assume that we're only checking for a full site scan
            // doing it per suggestion scan is just too small to warrant asking the user if he wants to confirm the purchase
            $processable = self::get_processable_post_ids();
            set_transient('wpil_ai_content_estimate_post_ids', $processable, 10 * MINUTE_IN_SECONDS);
            $total = count($processable);
            set_transient('wpil_ai_content_estimate_post_id_count', $total, 10 * MINUTE_IN_SECONDS);
        }

        // first, figure out what we're doing
        $active_processes = Wpil_Settings::get_selected_ai_batch_processes(true);

        $processes = array();
        foreach($active_processes as $process){
            if(Wpil_AI::check_batch_status_completed($process)){
                continue;
            }

            $model = Wpil_Settings::getChatGPTVersion($process);
            $processes[$model] = true;
        }

        foreach($processable as $key => $post_id){
            // exist if we're over the limit
            if(Wpil_Base::overTimeLimit(0, 10)){
                break;
            }

            $bits = explode('_', $post_id);
            unset($processable[$key]);
            if(empty($bits)){
                continue;
            }

            $post = new Wpil_Model_Post($bits[1], $bits[0]);
            $content = $post->getContent();
            foreach($processes as $model => $process){
                $cost += ($mode === 'link-whisper') ? self::estimate_processing_token_cost($content, $model): self::estimate_processing_cost($content, $model);
            }
        }

        // update the counter in case we need to go around again
        set_transient('wpil_ai_content_estimate_post_ids', $processable, 10 * MINUTE_IN_SECONDS);
        set_transient('wpil_ai_content_estimate_post_id_count', $total, 10 * MINUTE_IN_SECONDS);
        set_transient('wpil_ai_content_estimate_cost', $cost, 10 * MINUTE_IN_SECONDS);

        $response = array(
            'finished' => (empty($processable)) ? 1: 0,
            'posts_remaining' => count($processable),
            'total' => $total,
            'cost' => $cost,
            'mode' => $mode
        );

        wp_send_json($response);
    }

    public static function ajax_get_wizard_credit_estimate(){
        Wpil_Base::verify_nonce('wpil_download_ai_data');

        $ai_linking_enabled = self::is_ai_linking_enabled_request();
        $process_key = get_option('wpil_ai_linking_process_key', null);

        wp_send_json(array(
            'success' => array(
                'ai_credits' => self::get_available_ai_credits(),
                'estimated_credit_cost' => self::estimate_site_processing_credit_cost($process_key, $ai_linking_enabled)
            )
        ));
    }

    private static function is_ai_linking_enabled_request(){
        if(isset($_POST['ai_linking_enabled'])){
            return !empty($_POST['ai_linking_enabled']);
        }

        return true;
    }

    public static function add_batch_cron_interval($schedules){
        if(!isset($schedules['hourly'])){
            $schedules['hourly'] = array(
                'interval' => 60 * 60,
                'display' => __('Hourly', 'wpil')
            );
        }
        return $schedules;
    }

    /**
     * 
     **/
    public static function schedule_batch_process(){
        if(!empty(Wpil_Settings::get_ai_batch_processing_active()) && !empty(Wpil_Settings::get_selected_ai_batch_processes())){
            if(!wp_get_schedule('wpil_ai_batch_process_cron')){
                wp_schedule_event(time(), 'hourly', 'wpil_ai_batch_process_cron');
            }
        }elseif(wp_get_schedule('wpil_ai_batch_process_cron')){
            self::clear_batch_process_cron();
        }
    }

    public static function clear_batch_process_cron(){
        $timestamp = wp_next_scheduled('wpil_ai_batch_process_cron');
        wp_unschedule_event($timestamp, 'wpil_ai_batch_process_cron');
    }

    /**
     * Runs and coordinates the the cron-based batch processes.
     **/
    public static function perform_cron_batch_process(){
        // don't run the cron task if there's a live download in process
        $live_download = get_transient('wpil_doing_ai_data_download');

        if(!empty($live_download) && ((int)$live_download + (MINUTE_IN_SECONDS * 5)) > time()){
            return;
        }

        // set a flag so that we know that we're downloading data
        set_transient('wpil_doing_ai_data_download', time(), MINUTE_IN_SECONDS * 3);

        $selected_processes = Wpil_Settings::get_selected_ai_batch_processes(true);

        // if no batches are selected
        if(empty($selected_processes)){
            // exit
            return;
        }

        // if we're running the AI service and there are no AI credits available
        if(Wpil_Settings::get_linkwhisper_ai_active() && empty(self::get_available_ai_credits(true))){ // TODO: handle the edge case where a customer will be out of AI credits, but there is AI relation & keyword data taht we can process
            return;
        }

        // first, do any needed batch processing
        self::process_batch_data();

        // if we could do all that in less than 20 seconds
        if(!Wpil_Base::overTimeLimit(5, 20)){

            // set the batch size limit
            self::$batch_limit = 50000;

            // queue up the possible batches
            if(in_array('create-post-embeddings', $selected_processes)){
                self::create_site_embeddings();
            }

            // if that took over 20 seconds
            if(Wpil_Base::overTimeLimit(5, 20)){
                // exist
                return;
            }

            if(
                in_array('post-summarizing', $selected_processes) || 
                in_array('product-detecting', $selected_processes) || 
                in_array('keyword-detecting', $selected_processes))
            {
                self::analyze_site_posts();
            }
        }

        // if there are no embeddings currently being processed and we still have time
        if(in_array('create-post-embeddings', $selected_processes) && !Wpil_Base::overTimeLimit(5, 35) && self::has_completed_post_embeddings() && !self::has_completed_post_embedding_calculations()){
            self::stepped_calculate_post_embeddings();
            // clear the old AI Sitemap
            Wpil_Sitemap::delete_sitemap(false, 'ai_sitemap');
        }

        // if we've completed the embedding calculations and we don't have the sitemap generated yet
        if(self::has_completed_post_embedding_calculations() && !Wpil_Sitemap::has_sitemap('ai_sitemap')){
            // generate it now
            $relatedness = Wpil_AI::calculate_relatedness_sitemap();
            Wpil_Sitemap::save_sitemap($relatedness, 'ai_sitemap', 'AI Sitemap');
        }

        // if the embeddingsd are complete and no other factors concern us
        if(self::has_completed_post_embedding_calculations()){
            // clear the embedding lock
            self::set_last_embedding_id_lock();
        }

        if(!Wpil_Base::overTimeLimit(5, 35)){
            self::do_post_save_finishing();

            if(!Wpil_Sitemap::has_sitemap('ai_product_sitemap') && self::check_batch_status_completed(3, true)){
                $products = Wpil_AI::calculate_product_sitemap();
                if(!empty($products)){
                    Wpil_Sitemap::save_sitemap($products, 'ai_product_sitemap', 'AI-Detected Product Sitemap');
                }
            }
        }
    }

    public static function set_free_subscription_check_wait($is_free = false){
        set_transient('wpil_is_free_ai_key', !empty($is_free), 10 * MINUTE_IN_SECONDS);
    }

    public static function get_free_subscription_check_wait(){
        return !empty(get_transient('wpil_is_free_ai_key'));
    }

    public static function set_if_free_oai_subscription($is_free = false){
        update_option('wpil_is_free_ai_key', !empty($is_free));
    }

    public static function is_free_oai_subscription($check = false){
        if($check && !empty(Wpil_Settings::getOpenAIKey()) && !empty(self::$ai)){
            $models = self::decode(self::$ai->listModels());

            if(!empty($models) && isset($models->data)){
                $non_free = array(
                    'gpt-4-turbo',
                    'gpt-4o',
                    'gpt-4'
                );
                $is_free = true;
                foreach($models->data as $model){
                    if($is_free && in_array($model->id, $non_free)){
                        $is_free = false;
                        break;
                    }
                }

                self::set_if_free_oai_subscription($is_free);
                return $is_free;
            }
        }

        return !empty(get_option('wpil_is_free_ai_key', 0));
    }

    public static function get_available_models(){
        if(self::$ai_service_connected){
            $supported_models = array(
                'gpt-4o' => 'GPT-4o', 
                'gpt-4o-mini' => 'GPT-4o Mini'
            );
            return $supported_models;
        }else{
            $supported_models = array(
                'gpt-4.1-mini' => 'GPT-4.1 Mini',
                'gpt-4.1-nano' => 'GPT-4.1 Nano',
                'gpt-4o' => 'GPT-4o', 
                'gpt-4o-mini' => 'GPT-4o Mini',
                'gpt-4-turbo' => 'GPT-4', 
                'gpt-3.5-turbo' => 'GPT-3.5'
            );
        }

        if(empty(Wpil_Settings::getOpenAIKey()) || empty(self::$ai)){
            return $supported_models;
        }

        $cached = get_transient('wpil_ai_listed_models');
        $free = self::is_free_oai_subscription();

        if($cached === 'no-models'){
            return array();
        }elseif(!empty($cached) && (!$free || $free && self::get_free_subscription_check_wait())){
            return array_intersect_key($supported_models, $cached);
        }

        $models = self::decode(self::$ai->listModels());

        if(!empty($models) && isset($models->data)){
            $non_free = array(
                'gpt-4-turbo',
                'gpt-4o',
                'gpt-4'
            );
            $model_ids = array();
            $is_free = true;
            foreach($models->data as $model){
                if(!isset($supported_models[$model->id])){
                    continue;
                }
                $model_ids[$model->id] = true;

                if($is_free && in_array($model->id, $non_free)){
                    $is_free = false;
                }
            }

            if($is_free){
                self::set_free_subscription_check_wait();
            }

            self::set_if_free_oai_subscription($is_free);
            set_transient('wpil_ai_listed_models', $model_ids, (3 * HOUR_IN_SECONDS));
            return array_intersect_key($supported_models, $model_ids);
        }else{
            set_transient('wpil_ai_listed_models', 'no-models', MINUTE_IN_SECONDS * 15);
        }

        return array();
    }

    public static function prepare_table(){
        global $wpdb;

        $ai_post_data           = $wpdb->prefix . "wpil_ai_post_data";
        $ai_product_data        = $wpdb->prefix . "wpil_ai_product_data";
        $ai_keyword_data        = $wpdb->prefix . "wpil_ai_keyword_data";
        $ai_token_table         = $wpdb->prefix . "wpil_ai_token_use_data";
        $embd_data_table        = $wpdb->prefix . "wpil_ai_embedding_data";
        $embd_calc_table        = $wpdb->prefix . "wpil_ai_embedding_calculation_data";
        $embd_phrase_table      = $wpdb->prefix . "wpil_ai_embedding_phrase_data";
        $embd_phrase_calc_table = $wpdb->prefix . "wpil_ai_embedding_phrase_calculation_data";
        $ai_sggstd_anchor_table = $wpdb->prefix . "wpil_ai_suggested_anchors";
        $ai_anchor_sntnce_table = $wpdb->prefix . "wpil_ai_processed_sentences";
        $ai_ignore_anchor_table = $wpdb->prefix . "wpil_ai_ignored_anchors"; // TODO: Check to see what our size && speed effect are while using the ai_phrase table for keeping track of ignored suggestions. If we need a separate lookup index, build this out.
        $batch_log_table        = $wpdb->prefix . "wpil_ai_batch_log";
        $error_log_table        = $wpdb->prefix . "wpil_ai_error_log";
        $system_error_log_table = $wpdb->prefix . "wpil_ai_system_error_log";
        $completed_log_table    = $wpdb->prefix . "wpil_ai_completed_batch_log";
        $ai_linking_table       = $wpdb->prefix . "wpil_ai_linking";

        // if the AI post data table doesn't exist
        $ai_tbl_exists = $wpdb->query("SHOW TABLES LIKE '{$ai_post_data}'");
        if(empty($ai_tbl_exists)){
            $ai_post_data_table_query = "CREATE TABLE IF NOT EXISTS {$ai_post_data} (
                                            ai_index bigint(20) unsigned NOT NULL AUTO_INCREMENT,
                                            post_id bigint(20) unsigned NOT NULL,
                                            post_type varchar(8),
                                            data_type tinyint(1) DEFAULT 1,
                                            summary longtext,
                                            process_time bigint(20),
                                            model_version varchar(168),
                                            PRIMARY KEY (ai_index),
                                            INDEX (post_id),
                                            INDEX (post_type)
                                        )";
                /**
                 * id === table index
                 * post_id === post|term id
                 * post_type === data type, 'post'|'term'
                 * data_type === boolint 'post' => 1|'term' => 0
                 * summary === AI summary describing the post|term
                 * process_time === timestamp of last process
                 * model_version === the AI model used to process the data
                 */

            // create DB table if it doesn't exist
            require_once (ABSPATH . 'wp-admin/includes/upgrade.php');
            dbDelta($ai_post_data_table_query);
        }

        // if the AI product data table doesn't exist
        $ai_prdct_tbl_exists = $wpdb->query("SHOW TABLES LIKE '{$ai_product_data}'");
        if(empty($ai_prdct_tbl_exists)){
            $ai_product_data_table_query = "CREATE TABLE IF NOT EXISTS {$ai_product_data} (
                                            ai_product_index bigint(20) unsigned NOT NULL AUTO_INCREMENT,
                                            post_id bigint(20) unsigned NOT NULL,
                                            post_type varchar(8),
                                            data_type tinyint(1) DEFAULT 1,
                                            products longtext,
                                            product_count int DEFAULT 0,
                                            process_time bigint(20),
                                            model_version varchar(168),
                                            PRIMARY KEY (ai_product_index),
                                            INDEX (post_id),
                                            INDEX (post_type)
                                        )";
                /**
                 * id === table index
                 * post_id === post|term id
                 * post_type === data type, 'post'|'term'
                 * data_type === boolint 'post' => 1|'term' => 0
                 * products === AI identified products within the post|term
                 * process_time === timestamp of last process
                 * model_version === the AI model used to process the data
                 */

            // create DB table if it doesn't exist
            require_once (ABSPATH . 'wp-admin/includes/upgrade.php');
            dbDelta($ai_product_data_table_query);
        }

        // if the AI keyword data table doesn't exist
        $ai_kwrd_tbl_exists = $wpdb->query("SHOW TABLES LIKE '{$ai_keyword_data}'");
        if(empty($ai_kwrd_tbl_exists)){
            $ai_keyword_data_table_query = "CREATE TABLE IF NOT EXISTS {$ai_keyword_data} (
                                            ai_keyword_index bigint(20) unsigned NOT NULL AUTO_INCREMENT,
                                            post_id bigint(20) unsigned NOT NULL,
                                            post_type varchar(8),
                                            data_type tinyint(1) DEFAULT 1,
                                            keywords longtext,
                                            keyword_count int DEFAULT 0,
                                            keywords_loaded tinyint(1) default 0,
                                            process_time bigint(20),
                                            model_version varchar(168),
                                            PRIMARY KEY (ai_keyword_index),
                                            INDEX (post_id),
                                            INDEX (post_type)
                                        )";
                /**
                 * id === table index
                 * post_id === post|term id
                 * post_type === data type, 'post'|'term'
                 * data_type === boolint 'post' => 1|'term' => 0
                 * keywords === AI identified products within the post|term
                 * process_time === timestamp of last process
                 * model_version === the AI model used to process the data
                 */

            // create DB table if it doesn't exist
            require_once (ABSPATH . 'wp-admin/includes/upgrade.php');
            dbDelta($ai_keyword_data_table_query);
        }

        // if the AI token data table doesn't exist
        $ai_tkn_tbl_exists = $wpdb->query("SHOW TABLES LIKE '{$ai_token_table}'");
        if(empty($ai_tkn_tbl_exists)){
            $ai_token_data_table_query = "CREATE TABLE IF NOT EXISTS {$ai_token_table} (
                                            token_index bigint(20) unsigned NOT NULL AUTO_INCREMENT,
                                            query_id varchar(128) DEFAULT NULL,
                                            transaction_type varchar(24) NOT NULL DEFAULT 'usage',
                                            transaction_ref varchar(128) DEFAULT NULL,
                                            transaction_note varchar(64) DEFAULT NULL,
                                            model_version varchar(168),
                                            batch_processed tinyint(1) DEFAULT 0,
                                            input_tokens int(10) unsigned NOT NULL DEFAULT 0,
                                            output_tokens int(10) unsigned NOT NULL DEFAULT 0,
                                            cached_prompt_tokens int(10) unsigned NOT NULL DEFAULT 0,
                                            reasoning_tokens int(10) unsigned NOT NULL DEFAULT 0,
                                            total_tokens int(10) unsigned NOT NULL DEFAULT 0,
                                            credits_used decimal(10,4) unsigned NOT NULL DEFAULT 0.0000,
                                            credits_added decimal(10,4) unsigned NOT NULL DEFAULT 0.0000,
                                            process_used int(10) unsigned NOT NULL DEFAULT 0,
                                            process_key varchar(64) NOT NULL DEFAULT '',
                                            process_id int(10) unsigned NOT NULL DEFAULT 0,
                                            process_time bigint(20),
                                            PRIMARY KEY (token_index),
                                            INDEX (query_id),
                                            INDEX (transaction_type),
                                            INDEX (transaction_ref),
                                            INDEX process_key_id_time (process_key, process_id, process_time),
                                            UNIQUE KEY credit_transaction_ref (transaction_type, transaction_ref)
                                        )";

            // create DB table if it doesn't exist
            require_once (ABSPATH . 'wp-admin/includes/upgrade.php');
            dbDelta($ai_token_data_table_query);
        }

        // if the AI embedding data table doesn't exist
        $emdb_data_tbl_exists = $wpdb->query("SHOW TABLES LIKE '{$embd_data_table}'");
        if(empty($emdb_data_tbl_exists)){
            $embd_data_table_query = "CREATE TABLE IF NOT EXISTS {$embd_data_table} (
                                            embed_index bigint(20) unsigned NOT NULL AUTO_INCREMENT,
                                            post_id bigint(20) unsigned NOT NULL,
                                            post_type varchar(8),
                                            data_type tinyint(1) DEFAULT 1,
                                            embed_data longtext,
                                            is_empty TINYINT(1) UNSIGNED NOT NULL DEFAULT 0,
                                            process_time bigint(20),
                                            model_version varchar(168),
                                            PRIMARY KEY (embed_index),
                                            INDEX (post_id),
                                            INDEX (post_type)
                                        )";
                /**
                 * id === table index
                 * post_id === post|term id
                 * post_type === data type, 'post'|'term'
                 * data_type === boolint 'post' => 1|'term' => 0
                 * summary === AI summary describing the post|term
                 * products === AI identified products within the post|term
                 * process_time === timestamp of last process
                 * model_version === the AI model used to process the data
                 */

            // create DB table if it doesn't exist
            require_once (ABSPATH . 'wp-admin/includes/upgrade.php');
            dbDelta($embd_data_table_query);
        }

        // if the embedding calculation data table doesn't exist
        $embd_calc_tbl_exists = $wpdb->query("SHOW TABLES LIKE '{$embd_calc_table}'");
        if(empty($embd_calc_tbl_exists)){
            $embd_calc_table_query = "CREATE TABLE IF NOT EXISTS {$embd_calc_table} (
                                            embed_index bigint(20) unsigned NOT NULL AUTO_INCREMENT,
                                            post_id bigint(20) unsigned NOT NULL,
                                            post_type varchar(8),
                                            data_type tinyint(1) DEFAULT 1,
                                            calculation longtext,
                                            calc_index bigint(20) unsigned NOT NULL DEFAULT 0,
                                            calc_count bigint(20) unsigned NOT NULL DEFAULT 0,
                                            process_time bigint(20),
                                            model_version varchar(168),
                                            PRIMARY KEY (embed_index),
                                            INDEX (post_id),
                                            INDEX (post_type),
                                            INDEX (calc_index)
                                        )";
                /**
                 * data_type === boolint 'post' => 1|'term' => 0
                 */

            // create DB table if it doesn't exist
            require_once (ABSPATH . 'wp-admin/includes/upgrade.php');
            dbDelta($embd_calc_table_query);
        }
        
        // if the AI phrase embedding data table doesn't exist
        $emdb_phrs_data_tbl_exists = $wpdb->query("SHOW TABLES LIKE '{$embd_phrase_table}'");
        if(empty($emdb_phrs_data_tbl_exists)){
            $embd_phrs_data_table_query = "CREATE TABLE IF NOT EXISTS {$embd_phrase_table} (
                                            embed_index bigint(20) unsigned NOT NULL AUTO_INCREMENT,
                                            post_id bigint(20) unsigned NOT NULL,
                                            post_type varchar(8),
                                            data_type tinyint(1) DEFAULT 1,
                                            post_phrase_id varchar(168),
                                            embed_data longtext,
                                            no_data tinyint(1) DEFAULT 0,
                                            process_time bigint(20),
                                            model_version varchar(168),
                                            dimension_count int(8) DEFAULT 0,
                                            PRIMARY KEY (embed_index),
                                            INDEX (post_id),
                                            INDEX (post_type)
                                        )";

            // create DB table if it doesn't exist
            require_once (ABSPATH . 'wp-admin/includes/upgrade.php');
            dbDelta($embd_phrs_data_table_query);
        }

        // if the AI phrase embedding calculating result data table doesn't exist
        $emdb_phrs_calc_data_tbl_exists = $wpdb->query("SHOW TABLES LIKE '{$embd_phrase_calc_table}'");
        if(empty($emdb_phrs_calc_data_tbl_exists)){
            $emdb_phrs_calc_data_table_query = "CREATE TABLE IF NOT EXISTS {$embd_phrase_calc_table} (
                                            embed_index bigint(20) unsigned NOT NULL AUTO_INCREMENT,
                                            post_id bigint(20) unsigned NOT NULL,
                                            post_type varchar(8),
                                            data_type tinyint(1) DEFAULT 1,
                                            post_phrase_id varchar(168),
                                            calculation longtext,
                                            calc_index longtext,
                                            calc_count bigint(20) unsigned NOT NULL DEFAULT 0,
                                            no_data tinyint(1) DEFAULT 0,
                                            process_time bigint(20),
                                            model_version varchar(168),
                                            dimension_count int(8) DEFAULT 0,
                                            PRIMARY KEY (embed_index),
                                            INDEX (post_id),
                                            INDEX (post_type)
                                        )";

            // create DB table if it doesn't exist
            require_once (ABSPATH . 'wp-admin/includes/upgrade.php');
            dbDelta($emdb_phrs_calc_data_table_query);
        }

        // if the AI suggested anchor data table doesn't exist
        $sggstd_nchr_tbl_exists = $wpdb->query("SHOW TABLES LIKE '{$ai_sggstd_anchor_table}'");
        if(empty($sggstd_nchr_tbl_exists)){
            $sggstd_nchr_table_query = "CREATE TABLE IF NOT EXISTS {$ai_sggstd_anchor_table} (
                                            suggestion_index bigint(20) unsigned NOT NULL AUTO_INCREMENT,
                                            post_id bigint(20) unsigned NOT NULL,
                                            post_type varchar(8),
                                            data_type tinyint(1) DEFAULT 1,
                                            sentence_post_id varchar(168),
                                            sentence_id varchar(168),
                                            suggestion_words text,
                                            notes text,
                                            target_id bigint(20) unsigned NOT NULL,
                                            target_type varchar(8),
                                            target_data_type tinyint(1) DEFAULT 1,
                                            link_score int(4),
                                            ignore_suggestion tinyint(1) DEFAULT 0,
                                            process_time bigint(20),
                                            model_version varchar(168),
                                            PRIMARY KEY (suggestion_index),
                                            INDEX (post_id),
                                            INDEX (post_type),
                                            INDEX (sentence_post_id)
                                        )";

            // create DB table if it doesn't exist
            require_once (ABSPATH . 'wp-admin/includes/upgrade.php');
            dbDelta($sggstd_nchr_table_query);
        }

        // if the table that keeps track if we've scanned post sentences or not doesn't exist
        $nchr_sntnc_tbl_exists = $wpdb->query("SHOW TABLES LIKE '{$ai_anchor_sntnce_table}'");
        if(empty($nchr_sntnc_tbl_exists)){
            $ai_anchor_sntnce_table_query = "CREATE TABLE IF NOT EXISTS {$ai_anchor_sntnce_table} (
                                            sentence_index bigint(20) unsigned NOT NULL AUTO_INCREMENT,
                                            post_id bigint(20) unsigned NOT NULL,
                                            post_type varchar(8),
                                            data_type tinyint(1) DEFAULT 1,
                                            has_link tinyint(1) DEFAULT 0,
                                            sentence_id varchar(168),
                                            process_time bigint(20),
                                            PRIMARY KEY (sentence_index),
                                            INDEX (post_id),
                                            INDEX (post_type),
                                            INDEX (sentence_id)
                                        )";

            // create DB table if it doesn't exist
            require_once (ABSPATH . 'wp-admin/includes/upgrade.php');
            dbDelta($ai_anchor_sntnce_table_query);
        }

        // if the AI post data table doesn't exist
        $batch_lg_tbl_exists = $wpdb->query("SHOW TABLES LIKE '{$batch_log_table}'");
        if(empty($batch_lg_tbl_exists)){
            $batch_log_table_query = "CREATE TABLE IF NOT EXISTS {$batch_log_table} (
                                            log_index bigint(20) unsigned NOT NULL AUTO_INCREMENT,
                                            batch_id varchar(128),
                                            batch_data longtext,
                                            process_id tinyint(1) DEFAULT 0,
                                            process_time bigint(20) UNSIGNED,
                                            check_time bigint(20) UNSIGNED,
                                            PRIMARY KEY (log_index),
                                            INDEX (batch_id)
                                        )";

            // create DB table if it doesn't exist
            require_once (ABSPATH . 'wp-admin/includes/upgrade.php');
            dbDelta($batch_log_table_query);
        }

        // if the AI post data table doesn't exist
        $rrr_lg_tbl_exists = $wpdb->query("SHOW TABLES LIKE '{$error_log_table}'");
        if(empty($rrr_lg_tbl_exists)){
            $rrr_lg_tbl_query = "CREATE TABLE IF NOT EXISTS {$error_log_table} (
                                            log_index bigint(20) unsigned NOT NULL AUTO_INCREMENT,
                                            post_id bigint(20) unsigned NOT NULL,
                                            post_type varchar(8),
                                            data_type tinyint(1) DEFAULT 1,
                                            batch_id varchar(128),
                                            batch_data longtext,
                                            message_text longtext,
                                            process_time bigint(20) UNSIGNED,
                                            PRIMARY KEY (log_index),
                                            INDEX (post_id),
                                            INDEX (post_type),
                                            INDEX (batch_id)
                                        )";

            // create DB table if it doesn't exist
            require_once (ABSPATH . 'wp-admin/includes/upgrade.php');
            dbDelta($rrr_lg_tbl_query);
        }

        // if the AI post data table doesn't exist
        $sstm_rrr_lg_tbl_exists = $wpdb->query("SHOW TABLES LIKE '{$system_error_log_table}'");
        if(empty($sstm_rrr_lg_tbl_exists)){
            $sstm_rrr_lg_tbl_query = "CREATE TABLE IF NOT EXISTS {$system_error_log_table} (
                                            log_index bigint(20) unsigned NOT NULL AUTO_INCREMENT,
                                            message_text longtext,
                                            log_data longtext,
                                            process_time bigint(20) UNSIGNED,
                                            PRIMARY KEY (log_index)
                                        )";

            // create DB table if it doesn't exist
            require_once (ABSPATH . 'wp-admin/includes/upgrade.php');
            dbDelta($sstm_rrr_lg_tbl_query);
        }

        // if the AI post data table doesn't exist
        $cmpltd_lg_tbl_exists = $wpdb->query("SHOW TABLES LIKE '{$completed_log_table}'");
        if(empty($cmpltd_lg_tbl_exists)){
            $cmpltd_lg_tbl_query = "CREATE TABLE IF NOT EXISTS {$completed_log_table} (
                                            log_index bigint(20) unsigned NOT NULL AUTO_INCREMENT,
                                            batch_id varchar(128),
                                            batch_status varchar(128),
                                            process_time bigint(20) UNSIGNED,
                                            PRIMARY KEY (log_index)
                                        )";

            // create DB table if it doesn't exist
            require_once (ABSPATH . 'wp-admin/includes/upgrade.php');
            dbDelta($cmpltd_lg_tbl_query);
        }

        // if the AI post data table doesn't exist
        $ai_lnk_tbl_exists = $wpdb->query("SHOW TABLES LIKE '{$ai_linking_table}'");
        if(empty($ai_lnk_tbl_exists)){
            $ai_linking_table_table_query = "CREATE TABLE IF NOT EXISTS {$ai_linking_table} (
                                            ai_index bigint(20) unsigned NOT NULL AUTO_INCREMENT,
                                            post_id bigint(20) unsigned NOT NULL,
                                            post_type varchar(8),
                                            data_type tinyint(1) DEFAULT 1,
                                            target_id bigint(20) unsigned NOT NULL,
                                            target_type varchar(8),
                                            target_data_type tinyint(1) DEFAULT 1,
                                            sentence_text text DEFAULT NULL,
                                            sentence_id varchar(168) DEFAULT '',
                                            sentence_with_anchor_text text DEFAULT NULL,
                                            ai_relation_score double UNSIGNED NOT NULL DEFAULT 0,
                                            ignored tinyint(1) DEFAULT 0,
                                            inserted tinyint(1) DEFAULT 0,
                                            process_key varchar(64) DEFAULT '',
                                            process_time bigint(20),
                                            model_version varchar(168),
                                            PRIMARY KEY (ai_index),
                                            INDEX (post_id),
                                            INDEX (post_type),
                                            INDEX (sentence_id),
                                            INDEX (process_key)
                                        )";
                /**
                 * id === table index
                 * post_id === post|term id
                 * post_type === data type, 'post'|'term'
                 * data_type === boolint 'post' => 1|'term' => 0
                 * summary === AI summary describing the post|term
                 * process_time === timestamp of last process
                 * model_version === the AI model used to process the data
                 */

            // create DB table if it doesn't exist
            require_once (ABSPATH . 'wp-admin/includes/upgrade.php');
            dbDelta($ai_linking_table_table_query);
        }
    }

    /**
     * 
     **/
    public static function clear_ai_data($params = []){
        global $wpdb;

        // NOTE: some tables are intentionally not being cleared until we get past teh initial rollout... We might need things like log data
        $tables = array();
//      $wpdb->prefix . "wpil_ai_token_use_data",
//      $wpdb->prefix . "wpil_ai_error_log",
//      $wpdb->prefix . "wpil_ai_system_error_log"

        if(empty($params)){
            $tables = array_merge($tables, [
                $wpdb->prefix . "wpil_ai_batch_log"
            ]);
        }

        if(empty($params) || in_array('post_summary', $params)){
            $tables = array_merge($tables, [
                $wpdb->prefix . "wpil_ai_post_data"
            ]);
        }

        if(empty($params) || in_array('product_analysis', $params)){
            $tables = array_merge($tables, [
                $wpdb->prefix . "wpil_ai_product_data"
            ]);
        }

        if(empty($params) || in_array('keyword_analysis', $params)){
            $tables = array_merge($tables, [
                $wpdb->prefix . "wpil_ai_keyword_data"
            ]);
        }

        if(empty($params) || in_array('relation_analysis', $params)){
            $tables = array_merge($tables, [
                $wpdb->prefix . "wpil_ai_embedding_data",
                $wpdb->prefix . "wpil_ai_embedding_calculation_data",
                $wpdb->prefix . "wpil_ai_embedding_phrase_data",
                $wpdb->prefix . "wpil_ai_embedding_phrase_calculation_data",
                $wpdb->prefix . "wpil_ai_suggested_anchors",
                $wpdb->prefix . "wpil_ai_processed_sentences"
            ]);
        }

        foreach($tables as $table){
            $table_exists = $wpdb->get_var("SHOW TABLES LIKE '{$table}'");
            if($table_exists === $table){
                $wpdb->query("TRUNCATE TABLE {$table}");
            }
        }

        if(empty($params) || in_array('keyword_analysis', $params)){
            // clear any AI keywords
            $target_keywords_table = $wpdb->prefix . "wpil_target_keyword_data";
            $wpdb->delete($target_keywords_table, array('keyword_type' => 'ai-generated-keyword'));
        }

        if(empty($params) || in_array('product_analysis', $params)){
            $product_sitemap = Wpil_Sitemap::get_sitemap_list('ai_product_sitemap');
            if(!empty($product_sitemap) && isset($product_sitemap[0], $product_sitemap[0]->sitemap_id)){
                Wpil_Sitemap::delete_sitemap($product_sitemap[0]->sitemap_id, $product_sitemap[0]->sitemap_type);
            }
        }

        if(empty($params) || in_array('relation_analysis', $params)){
            // clear any AI created sitemaps
            $ai_sitemap = Wpil_Sitemap::get_sitemap_list('ai_sitemap');
            if(!empty($ai_sitemap) && isset($ai_sitemap[0], $ai_sitemap[0]->sitemap_id)){
                Wpil_Sitemap::delete_sitemap($ai_sitemap[0]->sitemap_id, $ai_sitemap[0]->sitemap_type);
            }
        }

        // for the time being, we'll just assume that everything worked.
        return true;
    }

    /**
     * Gets the ids of all logged batches
     **/
    public static function get_batch_log_ids(){
        global $wpdb;
        $table = $wpdb->prefix . 'wpil_ai_batch_log';

        return $wpdb->get_col("SELECT `batch_id` FROM {$table} ORDER BY `check_time` DESC");
    }

    /**
     * Gets the ids of all logged batches
     **/
    public static function get_next_process_batch_log_id(){
        global $wpdb;
        $table = $wpdb->prefix . 'wpil_ai_batch_log';

        return $wpdb->get_col("SELECT `batch_id` FROM {$table} ORDER BY `check_time` DESC LIMIT 1");
    }

    public static function get_batch_log_data($batch_id = '', $process = 0, $all = false){
        global $wpdb;
        $table = $wpdb->prefix . 'wpil_ai_batch_log';

        if(empty($batch_id) && empty($process) && empty($all)){
            return false;
        }

        if(!empty($batch_id)){
            return $wpdb->get_row($wpdb->prepare("SELECT * FROM {$table} WHERE `batch_id` = %s", $batch_id));
        }else{
            if(!empty($all)){
                return $wpdb->get_results("SELECT * FROM {$table}");
            }else{
                if(is_string($process)){
                    $process = self::get_process_code_from_name($process);
                }
                return $wpdb->get_results($wpdb->prepare("SELECT * FROM {$table} WHERE `process_id` = %d", $process));
            }
        }
    }

    public static function save_batch_log_data($batch_id = '', $data = '', $process = 0){
        global $wpdb;
        $table = $wpdb->prefix . 'wpil_ai_batch_log';

        if(empty($batch_id) || !empty(self::get_batch_log_data($batch_id))){
            return false;
        }

        if(!is_string($data)){
            $data = Wpil_Toolbox::json_compress($data);
        }

        if(is_string($process)){
            $process = self::get_process_code_from_name($process);
        }

        $insert = $wpdb->insert($table, [
            'batch_id' => $batch_id,
            'batch_data' => $data,
            'process_id' => $process,
            'process_time' => time(),
        ]);

        return (!empty($insert)) ? $wpdb->insert_id: false;
    }

    /**
     * 
     **/
    public static function save_completed_batch_log_entry($batch_id, $batch_status = ''){
        global $wpdb;
        $completed_log_table = $wpdb->prefix . "wpil_ai_completed_batch_log";

        if(empty($batch_id) || empty($batch_status)){
            return false;
        }
    
        // if this batch hasn't already been 
        $logged = self::get_completed_batch_log_entries($batch_id);
        if(empty($logged)){
            $wpdb->insert(
                $completed_log_table, 
                array(  
                    'batch_id' => $batch_id, 
                    'batch_status' => $batch_status, 
                    'process_time' => time()
                ), 
                array('%s', '%s', '%d')
            );
        }
    
        // Return the ID of the inserted row
        return (!empty($logged)) ? $logged->log_index: $wpdb->insert_id;
    }

    /**
     * Cleans up old batch log entries so they don't pile up in the database
     **/
    public static function delete_old_completed_batch_log_entries(){
        global $wpdb;
        $completed_log_table = $wpdb->prefix . "wpil_ai_completed_batch_log";
    
        // Calculate the timestamp for 2 weeks ago
        $two_weeks_ago = time() - (14 * DAY_IN_SECONDS);
    
        $wpdb->query($wpdb->prepare("DELETE FROM {$completed_log_table} WHERE process_time < %d", $two_weeks_ago));
    }

    /**
     * 
     **/
    public static function get_completed_batch_log_entries($batch_id = null){
        global $wpdb;
        $completed_log_table = $wpdb->prefix . "wpil_ai_completed_batch_log";

        if($batch_id !== null){
            if(empty(self::$complete_log_cache)){
                $logs = self::get_completed_batch_log_entries();
                if(!empty($logs)){
                    foreach($logs as $log){
                        if(!isset(self::$complete_log_cache[$log->batch_id])){
                            self::$complete_log_cache[$log->batch_id] = $log;
                        }
                    }
                }
            }

            // Retrieve a specific entry by batch id
            $result = isset(self::$complete_log_cache[$batch_id]) ? self::$complete_log_cache[$batch_id]: array();
        }else{
            // Retrieve all entries sorted by process_time descending
            $result = $wpdb->get_results("SELECT * FROM {$completed_log_table} ORDER BY process_time DESC");
        }

        return $result;
    }

    /**
     * Updates the last time a specific batch was checked
     **/
    public static function update_batch_process_check_time($batch_id = ''){
        global $wpdb;
        $table = $wpdb->prefix . 'wpil_ai_batch_log';

        if(empty($batch_id) || !is_string($batch_id)){
            return false;
        }

        $wpdb->update($table, ['check_time' => time()], ['batch_id' => $batch_id]);
    }

    public static function delete_batch_log_data($batch_id = ''){
        global $wpdb;
        $table = $wpdb->prefix . 'wpil_ai_batch_log';

        if(empty($batch_id)){
            return false;
        }

        $deleted = $wpdb->delete($table, [
            'batch_id' => $batch_id
        ]);

        return !empty($deleted);
    }

    /**
     * 
     **/
    public static function track_error($error = ''){
        if(empty($error) || !is_string($error)){
            return false;
        }

        if(!isset(self::$error_log[$error])){
            self::$error_log[$error] = 0;
        }

        self::$error_log[$error] += 1;
        self::$current_error = $error;
    }

    /**
     * 
     **/
    public static function has_error($error = ''){
        if(empty($error) || !is_string($error)){
            return false;
        }

        if(!isset(self::$error_log[$error]) && !empty(self::$error_log[$error])){
            return true;
        }

        return false;
    }

    /**
     * Checks to see if the current error is one that should only be logged once, and it's already been logged at least once
     **/
    public static function check_single_log_error($error = '', $limit = 1){
        if(empty($error) || !is_string($error)){
            if(!empty(self::$current_error)){
                $error = self::$current_error;
            }else{
                return false;
            }
        }

        $single_log_errors = array(
            'rate_limit_exceeded',
            'insufficient_quota',
            'invalid_request_error',
            'invalid_api_key'
        );

        if(isset(self::$error_log[$error]) && !empty(self::$error_log[$error]) && in_array($error, $single_log_errors, true) && self::$error_log[$error] > $limit){
            return true;
        }

        return false;
    }

    /**
     * Saves error messages for specific posts that have been processed in a batch.
     * Does not handle batch-level errors
     * If a batch contains post A, B, C, and post B has an error...
     * The error data for post B is what this will save.
     **/
    public static function save_error_log_data($batch_id = '', $data = array(), $process = 0){
        global $wpdb;
        $table = $wpdb->prefix . 'wpil_ai_error_log';

        if(empty($batch_id) || empty($data) || self::check_single_log_error()){
            return false;
        }

        if(!is_array($data)){
            $data = array($data);
        }

        $total_count = 0;
        $count = 0;
        $insert_query = "INSERT INTO {$table} (post_id, post_type, data_type, batch_id, batch_data, message_text, process_time) VALUES ";
        $error_data = array();
        $place_holders = array();
        $total = count($data);
        $limit = 1000;
        foreach($data as $key => $dat){
            $total_count++;
            $dat = self::decode($dat);

            if( empty($dat) ||                              // if there's no data
                !isset(                                     // or we don't have all the data from OAI that we need
                    $dat->id,
                    $dat->response,
                    $dat->custom_id,
                    $dat->response->status_code,
                    $dat->response->body)
            ){
                continue;
            }

            $message = '';
            if( isset($dat->response->body->error, $dat->response->body->error->message) && 
                !empty($dat->response->body->error->message)
            ){
                $message = $dat->response->body->error->message;
            }

            if( isset($dat->response->body->body, $dat->response->body->body->error) && 
                is_string($dat->response->body->body->error) && !empty($dat->response->body->body->error)
            ){
                $message = $dat->response->body->body->error;
            }

            if(empty($message)){
                continue;
            }

            $ids = explode('_', $dat->custom_id);
            array_push(
                $error_data, 
                $ids[1],
                $ids[0],
                (($ids[0] === 'post') ? 1: 0),
                $dat->id,
                Wpil_Toolbox::json_compress($dat->response),
                $message,
                time()
            );
            $place_holders[] = "('%d', '%s', '%d', '%s', '%s', '%s', '%d')";

            // if we've hit the limit
            if($count > $limit || ($key + 1) >= $total){
                // assemble the insert
                $insert = ($insert_query . implode(', ', $place_holders));
                $insert = $wpdb->prepare($insert, $error_data);
                // insert the data
                $wpdb->query($insert);
                // reset the data variables
                $error_data = [];
                $place_holders = [];
                $count = 0;
            }

            $count++;
        }

        // if we still have data that hasn't been inserted
        if(!empty($error_data) && !empty($place_holders)){
            // assemble the insert
            $insert = ($insert_query . implode(', ', $place_holders));
            $insert = $wpdb->prepare($insert, $error_data);
            // and insert the data
            $wpdb->query($insert);
        }

        // unset the current error since it's been logged
        self::$current_error = false;

        // return the total number of posts processed
        return $total_count;
    }

    /**
     * Gets the error log data for specific post processing attempts
     **/
    public static function get_error_log_data(){
        global $wpdb;
        $table = $wpdb->prefix . 'wpil_ai_error_log';

        // TODO: Fill out if needed
    }

    /**
     * Saves system error messages from OpenAI.
     * Does not handle batch-level or post errors
     * @param object $data The error respose data from OAI
     **/
    public static function save_system_error_log_data($data = array(), $error_message = ''){
        global $wpdb;
        $table = $wpdb->prefix . 'wpil_ai_system_error_log';

        if(empty($data)){
            return false;
        }

        if(!empty($error_message)){
            $error_message = sanitize_text_field($error_message);
        }else{
            $error_message = (
                isset($data->error) && 
                !empty($data->error) && 
                isset($data->error->message) && 
                !empty($data->error->message)
            ) ? sanitize_text_field($data->error->message): 'An unknown error has occurred.';
        }


        $insert = $wpdb->insert($table, [
            'message_text' => $error_message,
            'log_data' => json_encode($data), // TODO: consider compressing
            'process_time' => time(),
        ]);

        return (!empty($insert)) ? $wpdb->insert_id: false;
    }

    /**
     * Gets a list of the recent system error log messages
     * Can be set to return X number of the most recent entries
     **/
    public static function get_system_error_log_data($entry_count = 0){
        global $wpdb;
        $table = $wpdb->prefix . 'wpil_ai_system_error_log';

        if(empty($entry_count)){
            $messages = $wpdb->get_results("SELECT * FROM {$table}");
        }else{
            $messages = $wpdb->get_results($wpdb->prepare("SELECT * FROM {$table} ORDER BY `log_index` DESC LIMIT %d", $entry_count));
        }

        return (!empty($messages)) ? $messages: array();
    }

    /**
     * Gets the combined post and system error logs
     **/
    public static function get_combined_error_logs($entry_count){
        global $wpdb;
        $system_table = $wpdb->prefix . 'wpil_ai_system_error_log';
        $post_table = $wpdb->prefix . 'wpil_ai_error_log';
        $messages = array();

        if(empty($entry_count)){
            $system_messages = $wpdb->get_results("SELECT * FROM {$system_table}");
            $post_messages = $wpdb->get_results("SELECT * FROM {$post_table}");
            $messages = array_merge($system_messages, $post_messages);
        }else{
            $system_messages = $wpdb->get_results($wpdb->prepare("SELECT * FROM {$system_table} ORDER BY `log_index` DESC LIMIT %d", $entry_count));
            $post_messages = $wpdb->get_results($wpdb->prepare("SELECT * FROM {$post_table} ORDER BY `log_index` DESC LIMIT %d", $entry_count));
            $messages = array_merge($system_messages, $post_messages);
        }

        if(!empty($messages)){
            usort($messages, function($a, $b){
                if ($a->process_time == $b->process_time) {
                    return 0;
                }

                return ($a->process_time < $b->process_time) ? 1 : -1;
            });
        }

        if(!empty($entry_count) && !empty($messages)){
            $messages = array_slice($messages, 0, $entry_count);
        }

        return (!empty($messages)) ? $messages: array();
    }

    /**
     * Gets a list of all the posts that have had their AI processes completed.
     * Includes the intermediate stats such as the embedding calculations and keywords assignations
     **/
    public static function get_completed_post_stats($return_count = false, $ignore_unselected = false){
        global $wpdb;
        
        if($ignore_unselected){
            $selected_processes = Wpil_Settings::get_selected_ai_batch_processes(true);
            $completed = array();
            $table_indexes = array();
            if(in_array('create-post-embeddings', $selected_processes)){
                $completed['create-post-embeddings'] = array();
                $completed['calculated-post-embeddings'] = array();
                $table_indexes['create-post-embeddings'] = $wpdb->prefix . "wpil_ai_embedding_data";
                $table_indexes['calculated-post-embeddings'] = $wpdb->prefix . "wpil_ai_embedding_calculation_data";
            }
            if(in_array('product-detecting', $selected_processes)){
                $completed['product-detecting'] = array();
                $table_indexes['product-detecting'] = $wpdb->prefix . "wpil_ai_product_data";
            }
            if(in_array('keyword-detecting', $selected_processes)){
                $completed['keyword-detecting'] = array();
                $table_indexes['keyword-detecting'] = $wpdb->prefix . "wpil_ai_keyword_data";
            }
        }else{
            $completed = array(
                'product-detecting' => array(),
                'create-post-embeddings' => array(),
                'calculated-post-embeddings' => array(),
                'keyword-detecting' => array(),
                'keyword-assigning' => array()
            );
    
            $table_indexes = array(
    //            'post-summarizing' => $wpdb->prefix . "wpil_ai_post_data",
                'product-detecting' => $wpdb->prefix . "wpil_ai_product_data",
                'create-post-embeddings' => $wpdb->prefix . "wpil_ai_embedding_data",
                'calculated-post-embeddings' => $wpdb->prefix . "wpil_ai_embedding_calculation_data",
                'keyword-detecting' => $wpdb->prefix . "wpil_ai_keyword_data",
            );
        }
        
        if(empty($completed)){
            return array();
        }

        $last_embedding_index = self::get_last_embedding_index();
        $processable = self::get_processable_post_ids(true);

        // get the completed posts
        foreach($table_indexes as $ind => $table){
            $keyword_processing = ($ind === 'keyword-detecting') ? true: false;
            $calculating_embeddings = ($ind === 'keyword-detecting') ? true: false;

            if($keyword_processing){
                $data = $wpdb->get_results("SELECT `post_id`, `post_type`, `keywords_loaded` FROM {$table}");
            }elseif($calculating_embeddings && !empty($last_embedding_index)){
                $data = $wpdb->get_results("SELECT `post_id`, `post_type` FROM {$table} WHERE `calc_index` >= {$last_embedding_index}");
            }else{
                $data = $wpdb->get_results("SELECT `post_id`, `post_type` FROM {$table}");
            }
            

            if(!empty($data)){
                foreach($data as $dat){
                    $id = $dat->post_type . '_' . $dat->post_id;

                    if(!isset($processable[$id])){
                        continue;
                    }

                    if(!isset($completed[$ind][$id])){
                        $completed[$ind][$id] = true;
                    }

                    if($keyword_processing && !empty($dat->keywords_loaded)){
                        if(!isset($completed['keyword-assigning'][$id])){
                            $completed['keyword-assigning'][$id] = true;
                        }
                    }
                }
            }
        }

        if($return_count && !empty($completed)){
            foreach($completed as $ind => $data){
                $completed[$ind] = count($data);
            }
        }

        return $completed;
    }

    /**
     * 
     **/
    public static function get_total_processable_posts(){
        return (count(Wpil_Report::get_all_post_ids('ai')) + count(Wpil_Report::get_all_term_ids()));
    }

    /**
     * Gets all of the post & term ids that are set for processing.
     **/
    public static function get_processable_post_ids($return_indexed = false){
        // get all of the post and term ids that we're set to process
        $post_ids = Wpil_Report::get_all_post_ids('ai');
        $term_ids = Wpil_Report::get_all_term_ids();
        $ids = array();

        if(!empty($post_ids)){
            foreach($post_ids as $post_id){
                $id = 'post_' . $post_id;
                $ids[$id] = true;
            }
        }

        if(!empty($term_ids)){
            foreach($term_ids as $term_id){
                $id = 'term_' . $term_id;
                $ids[$id] = true;
            }
        }

        return (!empty($ids) && !$return_indexed) ? array_keys($ids) : $ids;
    }

    /**
     * Gets a list of all the posts and their known stati
     **/
    public static function get_ai_batch_processing_status($return_count = true, $ignore_cache = false){

        if(!empty(self::$status_cache) && !$ignore_cache){
            $stati = self::$status_cache;
            if($return_count){
                // count all the items in the in progress logs 
                foreach(self::$status_cache as $state => $data){
                    if(!is_array($data) && !is_object($data)){
                        $stati[$state] = $data;
                        continue;
                    }
                    foreach($data as $process => $dat){
                        if(!isset($stati[$state])){
                            $stati[$state] = array();
                        }

                        if(empty($dat)){
                            $stati[$state][$process] = 0;
                        }elseif(is_array($dat)){
                            $stati[$state][$process] = count($dat);
                        }
                    }
                }
            }
            return $stati;
        }

        $indexes = array(
            'post-summarizing' => array(),
            'product-detecting' => array(),
            'create-post-embeddings' => array(),
            'calculated-post-embeddings' => array(),
            'keyword-detecting' => array(),
            'keyword-assigning' => array(),
        );
        $stati = array(
            'completed' => $indexes,
            'in_progress' => $indexes,
            'errored' => $indexes,
            'total' => $indexes
        );

        $completed = self::get_completed_post_stats();

        if(!empty($completed)){
            $stati['completed'] = $completed;
        }

        $log_data = self::get_batch_log_data(false, false, true);

        // if there are posts currently in a batch process
        if(!empty($log_data)){
            // go over each process
            foreach($log_data as $batch){
                // decompress the specific post data
                $dat = Wpil_Toolbox::json_decompress($batch->batch_data);
                // if that worked
                if(!empty($dat)){
                    // get the process name
                    $process = self::get_process_name_from_code($batch->process_id);

                    // if this is a single process
                    if(isset($stati['in_progress'][$process])){
                        $stati['in_progress'][$process] = array_unique(array_merge($stati['in_progress'][$process], $dat));
                    }else{
                        if(false !== strpos($process, 'summary')){
                            $stati['in_progress']['post-summarizing'] = array_unique(array_merge($stati['in_progress']['post-summarizing'], $dat));
                        }

                        if(false !== strpos($process, 'product')){
                            $stati['in_progress']['product-detecting'] = array_unique(array_merge($stati['in_progress']['product-detecting'], $dat));
                        }

                        if(false !== strpos($process, 'keyword')){
                            $stati['in_progress']['keyword-detecting'] = array_unique(array_merge($stati['in_progress']['keyword-detecting'], $dat));
                        }
                    }
                }
            }
        }

        // get all of the post and term ids that we're set to process
        $stati['total'] = self::get_total_processable_posts();

        self::$status_cache = $stati;

        if($return_count){
            // count all the items in the in progress logs 
            foreach($stati as $state => $data){
                if(!is_array($data) && !is_object($data)){
                    $stati[$state] = $data;
                    continue;
                }
                foreach($data as $process => $dat){
                    if(empty($dat)){
                        $stati[$state][$process] = 0;
                    }elseif(is_array($dat)){
                        $stati[$state][$process] = count($dat);
                    }
                }
            }
        }

        return $stati;
    }

    /**
     * 
     **/
    public static function check_batch_status_completed($process = '', $ignore_cache = false){
        if(empty($process)){
            return false;
        }
        
        $stati = self::get_ai_batch_processing_status(false, $ignore_cache);

        if(is_numeric($process)){
            $process = self::get_process_name_from_code($process);
        }

        $processable = self::get_processable_post_ids();
        $total = count($processable);

        if(isset($stati['completed'][$process]) && !empty($total)){
            $complete = 0;
            foreach($processable as $id){
                if(isset($stati['completed'][$process][$id])){
                    $complete++;
                }
            }

            return $complete >= $total;
        }

        return false;
    }

    /**
     * 
     **/
    public static function get_batch_status_completion_percent($process = '', $ignore_cache = false){
        if(empty($process)){
            return 0;
        }
        
        $stati = self::get_ai_batch_processing_status(false, $ignore_cache);

        if(is_numeric($process)){
            $process = self::get_process_name_from_code($process);
        }

        $processable = self::get_processable_post_ids();
        $total = count($processable);

        if(isset($stati['completed'][$process])){
            $complete = 0;
            foreach($processable as $id){
                if(isset($stati['completed'][$process][$id])){
                    $complete++;
                }
            }

            return (empty($complete)) ? 0: floor(($complete/intval($total)) * 100);
        }

        return 0;
    }

    /**
     * Checks to see if there is any AI processed data stored on the site
     **/
    public static function has_ai_processed_data($specific_table = ''){
        global $wpdb;

        $tables = array(
            $wpdb->prefix . "wpil_ai_post_data",
            $wpdb->prefix . "wpil_ai_product_data",
            $wpdb->prefix . "wpil_ai_keyword_data",
            $wpdb->prefix . "wpil_ai_embedding_data",
            $wpdb->prefix . "wpil_ai_embedding_calculation_data",
            $wpdb->prefix . "wpil_ai_batch_log",
//            $wpdb->prefix . "wpil_ai_error_log",
        );

        $has_data = false;
        foreach($tables as $table){
            // if we're looking for a specific table and this isn't it
            if(!empty($specific_table) && false === strpos($table, $specific_table)){
                // skip to the next one
                continue;
            }
            $has_data = !empty($wpdb->get_var("SELECT COUNT(*) FROM {$table} LIMIT 1"));
            if(!empty($has_data)){
                break;
            }
        }

        return $has_data;
    }

    /**
     * 
     **/
    public static function supports_structured_response($model = '', $batch = false){
        $model = strtolower($model);
        
        if($batch){
            return ($model !== 'gpt-4-turbo' && $model !== 'gpt-3.5-turbo' && $model !== 'gpt-4o') ? true: false;
        }else{
            return ($model !== 'gpt-4-turbo' && $model !== 'gpt-3.5-turbo') ? true: false;
        }
    }

    /**
     * 
     **/
    public static function create_response_format($name = 'batch_response', $data = array(), $model = '', $array_processing = false){
        if(empty($data) || !self::supports_structured_response($model, $name === 'batch_process')){
            return array('type' => 'json_object');
        }

        $properties = array();

        foreach($data as $ind => $type){
            $properties[$ind] = array(
                'type' => $type
            );
        }

        if(empty($array_processing)){
            $response_format = array(
                'type' => 'json_schema',
                'json_schema' => array(
                    'name' => $name,
                    'strict' => true,
                    'schema'=> array(
                        'type' => 'object',
                        'properties' => array(
                            'results' => array(
                                'type' => 'object',
                                'properties' => $properties,
                                'required' => array_keys($data),
                                'additionalProperties' => false
                            ),
                        ),
                        'required' => ['results'],
                        'additionalProperties' => false
                    )
                )
            );
        }else{
            $response_format = array(
                'type' => 'json_schema',
                'json_schema' => array(
                    'name' => $name,
                    'strict' => true,
                    'schema'=> array(
                        'type' => 'object',
                        'properties' => array(
                            'results' => array(
                                "type" => "array",
                                'items' => array(
                                    "type" => "object",
                                    "properties" => $properties,
                                    "required" => array_keys($data),
                                    'additionalProperties' => false
                                ),
                            )
                        ),
                        'required' => ['results'],
                        'additionalProperties' => false
                    )
                )
            );
        }

        return $response_format;
    }

    /**
     * 
     **/
    public static function create_response_indexes($purpose = ''){
        $indexes = array();
        if(empty($purpose)){
            return $indexes;
        }

        if($purpose === 'post-summarizing' || false !== strpos($purpose, 'summary')){
            $indexes["summary"] = "string";
        }
        
        if(false !== strpos($purpose, 'product')){
            $indexes["products"] = "string";
            $indexes["product-count"] = "integer";
        }

        if(false !== strpos($purpose, 'keyword')){
            $indexes["keywords"] = "string";
            $indexes["keyword-count"] = "integer";
        }

        if(false !== strpos($purpose, 'assess-sentence-anchors')){
            $indexes["meta_id"] = "string";
            $indexes["words"] = "string";
            $indexes["sentence"] = "string";
            $indexes["title"] = "string";
            $indexes["keywords"] = "string";
            $indexes["match_score"] = "integer";
            $indexes["free_form"] = "string";
        }

        return $indexes;
    }

    /**
     * Determines how related a list of sentences ($phrases) are to a slice of target text
     **/
    public static function determine_relatedness($phrases){
        /* INPUT FORMAT
            'sentence' => "what is doing the relating"
            'context' => "a larger slice of text that contains the 'sentence' to help with checking relatedness"
            'target => "what were comparing the relation to"
        */

        if(empty($phrases)){
            return array();
        }


        $prompt = array(
            'GOAL: Find all sentences that are related to the target text OR directly refers to the target text\'s subject. Follow each step and obey all of the rules while doing this.',
            'STEP 1: Examine the target text to understand what it refers to, its topic and subject.',
            'STEP 2: Examine the sentences individually and determine each sentence\'s topic and subject. Use the supplied "context" for additional information about the sentence.', 
            'STEP 3: Evaluate each sentence to see if it refers to the target text or is related to it.',
            'RULE: Sentences are deliminated by triple pipes (|||) followed by a sentence number',
            'RULE: Treat each sentence individually and do not use them for context to understand other sentences. The sentences are not related to each other, must only be compared to the target text.',
            'RULE: There are 14 sentences, read all of them.',
        //    'RULE: Sentences can be related to the target text if they have the same adjective-noun use as the target text (EX: "best blue shoes" is related to "best red shoes")',
            'RULE: List the sentence numbers for all sentences related to the target text and why they are related in JSON using this format: {"sentence number": ["reason related", "similarity score"]}. The "reason related" text should be longer than 3 words. The "similarity score" is a measure from 0 to 10 of how closely the sentence relates to the target, with 0 being not related to 10 meaning completely related.',
            'RULE: After the JSON response, state how many sentences you read and compared to the text.',
        //    'RULE: List the sentence numbers for all related sentences and why they are related in JSON using this format: {"sentence number": "reason related"}',
            //    'Do not respond with anything other than JSON and ignore sentences only containing numbers'
        );

        $prompt = array(
            'GOAL: Find all sentences that are related to the target text OR directly refers to the target text\'s subject. Follow each step and obey all of the rules while doing this.',
            'STEP 1: Examine the target_text to understand 1.)what it refers to, 2.) its topic and 3.) its subject.',
            'STEP 2: Examine each sentence_text individually and determine each sentence\'s topic and subject. Use the supplied "context" for additional information about the sentence.', 
            'STEP 3: Evaluate each sentence to see if it refers to the target text or is related to it.',
//            'RULE: Sentences are deliminated by triple pipes (|||) followed by a sentence number',
            'RULE: Treat each sentence individually and do not use them for context to understand other sentences. The sentences are not related to each other, must only be compared to the target text.',
//            'RULE: There are 14 sentences, read all of them.',
        //    'RULE: Sentences can be related to the target text if they have the same adjective-noun use as the target text (EX: "best blue shoes" is related to "best red shoes")',
            'RULE: List the sentence numbers for all sentences related to the target text and why they are related in JSON using this format: {"sentence number": ["reason related", "similarity score"]}. The "reason related" text should be longer than 3 words. The "similarity score" is a measure from 0 to 10 of how closely the sentence relates to the target, with 0 being not related to 10 meaning completely related.',
            'RULE: After the JSON response, state how many sentences you read and compared to the text.',
        //    'RULE: List the sentence numbers for all related sentences and why they are related in JSON using this format: {"sentence number": "reason related"}',
            //    'Do not respond with anything other than JSON and ignore sentences only containing numbers'
        );


        $questions = 0;
        $question = array();
        $question_len = 0;
        $question_list = array();
        foreach($phrases as $data){
            $id = isset($data['batch_key']) ? ($data['batch_key'] . '-'): '';
            $id .= isset($data['phrase_key']) ? ($data['phrase_key'] . '-'): '';
            $id .= isset($data['suggestion_key']) ? ($data['suggestion_key'] . '-'): '';
            $id = trim($id, '-');

            $q = array(
                'target_text' => $data['target'],
                'sentence_text' => $data['sentence'],
                'context' => $data['context'],
                'id' => $id
            );

            //$sentence = "|||" . $q['id'] . " target_text:" . $q['target_text'] . " sentence_text:" . $q['sentence_text'] . " context:" . $data['context'] . " ";
            $sentence = json_encode(array('block_id' => $q['id'], "target_text" => $q['target_text'], "sentence_text" => $q['sentence_text'], "context" => $data['context']));
            
            if(empty($sentence)){
                continue;
            }

            if((strlen($sentence) + $question_len) > self::$question_limit || $questions >= 10){
                $question_list[] = $question;
                $questions = 0;
                $question = array();
                $question_len = 0;
            }

            $questions++;
            $question[] = ("|||" . $sentence);
            $question_len += mb_strlen($sentence);
        }

        if(!empty($question)){
            $question_list[] = $question;
        }

        $message_list = array();
        $longest_question = 0;
        foreach($question_list as $questions){
            $county = count($questions);
            $prompt = array(
                'GOAL: Examine the data blocks to find if the block\'s sentences are related to their target text OR directly refers to their target text\'s subject. Follow each step and obey all of the rules while doing this.',
                'STEP 1: Examine the block\'s target_text to understand 1.)what it refers to, 2.) its topic and 3.) its subject.',
                'STEP 2: Examine each block\'s sentence_text individually and determine the sentence\'s topic and subject. Use the supplied "context" for additional information about the block\'s sentence_text.', 
                'STEP 3: Evaluate each block\'s sentence_text to see if it refers to the block\'s target text or is related to it.',
                'RULE: Data blocks are deliminated by triple pipes (|||) followed by a block id. All block data is JSON',
                'RULE: Treat each block individually and do not use them for context to understand other blocks. The blocks are not related to each other, and their sentence_text must only be compared to their target_text.',
                'RULE: Assume each block\'s sentence_text is separate and distinct from the block\'s target_text.',
                "RULE: There are {$county} blocks, read and process all of them.",
            //    'RULE: Sentences can be related to the target text if they have the same adjective-noun use as the target text (EX: "best blue shoes" is related to "best red shoes")',
            //    'RULE: List the results of your work in JSON using this format: {"block_id": {"related": "relation", "similarity_score": "score", "explanation": "rationale"}}. The "relation" status should be "yes" if a block\'s sentence_text is related to its block\'s target_text and "no" if not related. The "score" is a measure from 0 to 10 of how closely the block\'s sentence_text relates to the block\'s target_text, with 0 being not related to 10 meaning completely related. The "rationale" is your reasoning on why the block\'s sentence_text is or is not related to the block\'s target_text and must contain both the sentence_text and the target_text while explaining.',
            );
            $prompt[] = 'RULE: Create a "rationale" as the basis of your reasoning on why the block\'s sentence_text is or is not related to the block\'s target_text and must contain both your assessment of what the sentence_text means and your assessment of the target_text means and why they are or are not related. The "relation" status must be "yes" if a block\'s sentence_text is related to its block\'s target_text based on the block\'s "rationale" or it must be "no" if a block\'s sentence_text is not related to its block\'s target_text based on the block\'s "rationale". The "score" is a measure from 0 to 10 of how closely the block\'s sentence_text relates to the block\'s target_text, with 0 being not related to 10 meaning completely related.';

            // todo create setting to allow users to say if they want to include the explination in the response
            if(true){
                $prompt[] = 'RULE: List the results of your work in JSON using this format: {"block_id": {"related": "relation", "similarity_score": "score", "explanation": "rationale"}}.';
            }else{
                $prompt[] = 'RULE: List the results of your work in JSON using this format: {"block_id": {"related": "relation", "similarity_score": "score"}}. Do not include the "rationale" in your response';
            }
//            $prompt[] = 'RULE: The "rationale" is your reasoning on why the block\'s sentence_text is or is not related to the block\'s target_text and must contain both your assessment of what the sentence_text means and your assessment of the target_text means and why they are or are not related while explaining. The "relation" status must be "yes" if a block\'s sentence_text is related to its block\'s target_text based on the block\'s "rationale" or it must be "no" if a block\'s sentence_text is not related to its block\'s target_text based on the block\'s "rationale". The "score" is a measure from 0 to 10 of how closely the block\'s sentence_text relates to the block\'s target_text, with 0 being not related to 10 meaning completely related.';
            //    'RULE: List the sentence numbers for all related sentences and why they are related in JSON using this format: {"sentence number": "reason related"}',
            $prompt[] = 'RULE: You must only reply with the JSON, do not say anything that is not part of the JSON object.';
            $prompt[] = 'NOTE: ' . time(); // cachebuster

            $prompt = implode(' ', $prompt);
            $question = json_encode($questions);

            if($maybe_longer = mb_strlen($question) > $longest_question){
                $longest_question = $maybe_longer;
            }

            $message_list[] = 
            ['messages' => [
                [
                    "role" => "system",
                    "content" => $prompt
                ],
                [
                    "role" => "user",
                    "content" => $question
                ]
                ]
            ];
        }

        $args = array(
            'model' => Wpil_Settings::getChatGPTVersion('suggestion-scoring'),
            'response_format' => ["type" => "json_object"],
            'message_list' => $message_list,
            'temperature' => 0,
            'frequency_penalty' => 0,
            'presence_penalty' => -2.0
        );

        $chat = self::$ai->chat($args, null, true);
        $finished = array();
        $no_response = false;
        foreach($chat as $chat_id => $dat){

            $info = self::$ai->getCURLInfo()[$chat_id];
        
            if(empty($dat) && (!empty($info) && isset($info['http_code']) && $info['http_code'] === 0)){
                $no_response = true;
            }

            $response = self::decode($dat);
            $results = false;
            if(!empty($response) && isset($response->choices)){
                $results = json_decode($response->choices[0]->message->content);
    
                if(empty($results)){
                    $results = self::attempt_recover_json($response->choices[0]->message->content, $response->choices[0]->finish_reason);
                }

                if(!empty($results) && !isset($results->error) && !isset($results->error->code)){
                    $finished = array_merge($finished, (array)$results);
                }
            }elseif(isset($response->error) && isset($response->error->code) && $response->error->code === 'rate_limit_exceeded'){
                self::$rate_limited = true;
            }

            self::save_response_tokens($dat, 'suggestion-scoring');
        }

        // if we have results, return an object of them. If after iterating over all the responses we have nothing and there was at least one 0 response code, say that there was no response so we can try it again.
        return !empty($finished) ? (object)$finished: (($no_response || self::$rate_limited) ? 'no-response': array());
    }

    /**
     * Attempts to pull useable JSON content out of a partially completed string 
     **/
    public static function attempt_recover_json($string, $finish_reason = false){
        // if the string is empty or isn't likely to be JSON
        if(empty($string) || false === strpos($string, '{')){
            // return the string
            return $string;
        }

        // first, make sure that the content isn't too unslashed
        $recovered = self::reslash_output_json($string);
        $maybe_recovered = json_decode($recovered);

        // if that's all it needed
        if(!empty($maybe_recovered)){
            // return the json
            return $maybe_recovered;
        }

        // if that didn't work, we might be looking at a partial string
        preg_match_all('/("[\d\-]+":\s\{[^{}]+?\})/', $recovered, $matches);

        // if we were able to pull something
        if(!empty($matches) && !empty($matches[0])){
            // apply formatting...
            $recovered = '{' . implode(',', $matches[0]) . '}';
            // try parsing
            $maybe_recovered = json_decode($recovered);
            // if that worked
            if(!empty($maybe_recovered)){
                // it's our new data!
                return $recovered;
            }
        }

        preg_match_all('/("[\S\s]*?"|[0-9]*):[\s]*("[\S\s]*?"|[0-9]*)/', $recovered, $matches);
        if(!empty($matches) && !empty($matches[0])){
            $recovered = '{' . implode(',', $matches[0]) . '}';
            $maybe_recovered = json_decode($recovered);

            // if that's all it needed
            if(!empty($maybe_recovered)){
                // return the json
                return $recovered;
            }
        }

        return $string;
    }

    public static function reslash_output_json($string){
        // first, make sure the slashing is correct
        $recovered = mb_ereg_replace(preg_quote('\\'), '', $string);
        preg_match_all('/"explanation": "(.*?)"}/', $recovered, $matches);

        $slashed = false;
        if(isset($matches[1]) && !empty($matches[1])){
            foreach($matches[1] as $match){
                $recovered = str_replace($match, str_replace(['"'], ['\\"'], $match), $recovered);
                $slashed = true;
            }
        }

        return ($slashed) ? $recovered: $string;
    }

    public static function save_response_tokens($response, $process_used = '', $is_batch = false, $fallback_query_id = '', $process_key = '', $process_id = 0){
        $saved = false;

        if(empty($response)){
            return $saved;
        }

        if(empty($process_used) && !empty(self::$purpose)){
            $process_used = self::$purpose;
        }

        $process_code = self::get_process_code_from_name($process_used);

        if(!is_array($response)){
            $response = array($response);
        }

        foreach($response as $index => $dat){
            $query_id = self::extract_query_id_from_response($dat);
            if(is_string($dat)){
                $dat = self::decode($dat);
                if(empty($query_id)){
                    $query_id = self::extract_query_id_from_response($dat);
                }
            }
            if(empty($query_id) && !empty($fallback_query_id)){
                $query_id = (string) $fallback_query_id;
            }
            if(empty($query_id) && isset(self::$query_ids[$index]) && !empty(self::$query_ids[$index])){
                $query_id = self::$query_ids[$index];
            }

            if(isset($dat->response) && !empty($dat->response)){
                $dat = $dat->response;
            }

            if(isset($dat->body) && !empty($dat->body)){
                $body = is_string($dat->body) ? self::decode($dat->body) : $dat->body;
                if(is_object($body) && isset($body->usage)){
                    $dat = $body;
                }
            }

            if(!empty($dat) && isset($dat->model) && !empty($dat->model) && isset($dat->usage) && is_object($dat->usage)){
                $usage = $dat->usage;

                // Support both Chat Completions and Responses-style usage keys.
                $input = 0;
                if(isset($usage->prompt_tokens)){
                    $input = (int) $usage->prompt_tokens;
                }elseif(isset($usage->input_tokens)){
                    $input = (int) $usage->input_tokens;
                }

                $output = 0;
                if(isset($usage->completion_tokens)){
                    $output = (int) $usage->completion_tokens;
                }elseif(isset($usage->output_tokens)){
                    $output = (int) $usage->output_tokens;
                }

                $total = isset($usage->total_tokens) ? (int) $usage->total_tokens : 0;

                $cached_prompt = 0;
                if(isset($usage->prompt_tokens_details, $usage->prompt_tokens_details->cached_tokens)){
                    $cached_prompt = (int) $usage->prompt_tokens_details->cached_tokens;
                }elseif(isset($usage->input_tokens_details, $usage->input_tokens_details->cached_tokens)){
                    $cached_prompt = (int) $usage->input_tokens_details->cached_tokens;
                }elseif(isset($usage->cached_tokens)){
                    $cached_prompt = (int) $usage->cached_tokens;
                }

                $reasoning = 0;
                if(isset($usage->completion_tokens_details, $usage->completion_tokens_details->reasoning_tokens)){
                    $reasoning = (int) $usage->completion_tokens_details->reasoning_tokens;
                }elseif(isset($usage->output_tokens_details, $usage->output_tokens_details->reasoning_tokens)){
                    $reasoning = (int) $usage->output_tokens_details->reasoning_tokens;
                }

                $credits_used = 0;
                if(isset($usage->credit_used)){
                    $credits_used = (float) $usage->credit_used;
                }elseif(isset($usage->credits_used)){
                    $credits_used = (float) $usage->credits_used;
                }

                // Some provider responses only report cached token details.
                if($input < 1 && $cached_prompt > 0){
                    $input = $cached_prompt;
                }

                if($total < 1 && ($input > 0 || $output > 0)){
                    $total = ($input + $output);
                }

                $status = self::save_token_reference($dat->model, $is_batch, $process_code, $input, $output, $total, $cached_prompt, $reasoning, $credits_used, $query_id, 'usage', '', '', 0, 0, $process_key, $process_id);
            
                if(!$saved && $status){
                    $saved = true;
                }

                if(!empty($credits_used)){
                    self::subtract_ai_credits($credits_used);
                }
            }
        }

        return $saved;
    }

    public static function save_token_reference($model = '', $batch_process = false, $process_number = 0, $input_tokens = 0, $output_tokens = 0, $total_tokens = 0, $cached_prompt_tokens = 0, $reasoning_tokens = 0, $credits_used = 0, $query_id = '', $transaction_type = 'usage', $transaction_ref = '', $transaction_note = '', $process_time = 0, $credits_added = 0, $process_key = '', $process_id = 0){
        global $wpdb;
        $table = $wpdb->prefix . 'wpil_ai_token_use_data';

        if(empty($model)){
            return false;
        }

        if(empty($total_tokens) && (!empty($input_tokens) || !empty($output_tokens))){
            $total_tokens = ($input_tokens + $output_tokens);
        }

        $context = array('task_key' => '', 'process_id' => 0);
        if($transaction_type === 'usage'){
            $context = self::get_current_credit_tracking_context($process_number, $batch_process, $process_key, $process_id);
        }

        $saved = $wpdb->insert($table, [
            'query_id' => !empty($query_id) ? substr(sanitize_text_field((string)$query_id), 0, 128) : null,
            'transaction_type' => !empty($transaction_type) ? substr(sanitize_key((string) $transaction_type), 0, 24) : 'usage',
            'transaction_ref' => !empty($transaction_ref) ? substr(sanitize_text_field((string) $transaction_ref), 0, 128) : null,
            'transaction_note' => !empty($transaction_note) ? substr(sanitize_text_field((string) $transaction_note), 0, 64) : null,
            'model_version' => $model,
            'batch_processed' => (int) $batch_process,
            'input_tokens' => $input_tokens,
            'output_tokens' => $output_tokens,
            'cached_prompt_tokens' => $cached_prompt_tokens,
            'reasoning_tokens' => $reasoning_tokens,
            'total_tokens' => $total_tokens,
            'credits_used' => $credits_used,
            'credits_added' => round((float) $credits_added, 4),
            'process_used' => (int) $process_number,
            'process_key' => !empty($context['task_key']) ? $context['task_key'] : '',
            'process_id' => !empty($context['process_id']) ? (int) $context['process_id'] : 0,
            'process_time' => !empty($process_time) ? (int) $process_time : time(),
        ]);

        return !empty($saved);
    }

    public static function save_credit_deposit_notice($purchase_id = '', $credits = 0, $purchase_type = '', $purchase_time = 0){
        global $wpdb;
        $table = $wpdb->prefix . 'wpil_ai_token_use_data';
        $purchase_id = substr(sanitize_text_field((string) $purchase_id), 0, 128);
        $credits = round((float) $credits, 4);
        $purchase_time = !empty($purchase_time) ? (int) $purchase_time : time();

        if(empty($purchase_id) || $credits <= 0){
            return false;
        }

        // If this purchase is already in the ledger, we can skip it.
        $existing = $wpdb->get_var($wpdb->prepare("SELECT token_index FROM {$table} WHERE `transaction_type` = %s AND `transaction_ref` = %s LIMIT 1", 'credit-deposit', $purchase_id));
        if(!empty($existing)){
            return true;
        }

        $saved = self::save_token_reference(
            'credit-deposit', false, self::get_process_code_from_name('credit-deposit'), 0, 0, 0, 0, 0, 0, '', 'credit-deposit', $purchase_id, $purchase_type, $purchase_time, $credits
        );

        if(!$saved){
            // If another request slipped the same purchase in first, count that as a win.
            $existing = $wpdb->get_var($wpdb->prepare("SELECT token_index FROM {$table} WHERE `transaction_type` = %s AND `transaction_ref` = %s LIMIT 1", 'credit-deposit', $purchase_id));
            return !empty($existing);
        }

        return true;
    }

    private static function extract_query_id_from_response($response = null){
        if(empty($response)){
            return '';
        }

        if(is_string($response)){
            $decoded = self::decode($response);
            if(!empty($decoded)){
                $response = $decoded;
            }else{
                return '';
            }
        }

        if(is_array($response)){
            $response = (object) $response;
        }

        if(!is_object($response)){
            return '';
        }

        if(isset($response->custom_id) && !empty($response->custom_id)){
            return (string) $response->custom_id;
        }

        if(isset($response->query_id) && !empty($response->query_id)){
            return (string) $response->query_id;
        }

        if(isset($response->metadata) && is_object($response->metadata)){
            if(isset($response->metadata->custom_id) && !empty($response->metadata->custom_id)){
                return (string) $response->metadata->custom_id;
            }
            if(isset($response->metadata->query_id) && !empty($response->metadata->query_id)){
                return (string) $response->metadata->query_id;
            }
        }

        return '';
    }

    /**
     * Calculates how much the user has spent on tokens based on a time range
     **/
    public static function calculate_token_cost_by_time($start_time = 0, $end_time = 0){
        global $wpdb;
        $table = $wpdb->prefix . 'wpil_ai_token_use_data';

        // As of 09-18-2024
        $normal_costs_per_model = self::get_standard_model_costs();

        $unprefixed = array(
            'gpt-4o-mini' => array('input' => 0.15/1000000, 'output' => 0.600/1000000),
            'gpt-4o' => array('input' => 5.00/1000000, 'output' => 15.00/1000000),
            'gpt-4-turbo' => array('input' => 10.00/1000000, 'output' => 30.00/1000000),
        );

        if(empty($start_time)){
            return 0;
        }

        if(empty($end_time)){
            $end_time = time();
        }

        // if we're using our AI service
        if(self::$ai_service_connected){
            // just pull the credits and return them
            $cost = $wpdb->get_var($wpdb->prepare("SELECT SUM(credits_used) FROM {$table} WHERE `process_time` >= %d AND `process_time` <= %d AND `transaction_type` = %s", $start_time, $end_time, 'usage'));
            return round((float) $cost, 4);
        }

        $tokens = $wpdb->get_results($wpdb->prepare("SELECT * FROM {$table} WHERE `process_time` >= %d AND `process_time` <= %d", $start_time, $end_time));
    
        $cost = 0;
        if(!empty($tokens)){
            foreach($tokens as $token){
                if(false !== strpos($token->model_version, 'text-embedding')){
                    $price = ($token->total_tokens * $normal_costs_per_model[$token->model_version]['input']);
                    $cost += ($token->batch_processed) ? ($price/2): $price;
                }elseif(isset($normal_costs_per_model[$token->model_version])){
                    $input = (!empty($token->cached_prompt_tokens)) ? abs($token->input_tokens - $token->cached_prompt_tokens): $token->input_tokens;
                    $price =  (
                        ($input * $normal_costs_per_model[$token->model_version]['input']) + 
                        ($token->output_tokens * $normal_costs_per_model[$token->model_version]['output']) +
                        ($token->cached_prompt_tokens * ($normal_costs_per_model[$token->model_version]['input']/2))
                    );

                    $cost += ($token->batch_processed) ? ($price/2): $price;
                }else{
                    foreach($unprefixed as $model => $dat){
                        if(false !== strpos($token->model_version, $model)){
                            $input = (!empty($token->cached_prompt_tokens)) ? abs($token->input_tokens - $token->cached_prompt_tokens): $token->input_tokens;
                            $price = (
                                ($input * $unprefixed[$model]['input']) + 
                                ($token->output_tokens * $unprefixed[$model]['output']) +
                                ($token->cached_prompt_tokens * ($unprefixed[$model]['input']/2))
                            );
                            $cost += ($token->batch_processed) ? ($price/2): $price;
                            break;
                        }
                    }
                }
            }
        }

        return $cost;
    }

    public static function get_standard_model_costs(){
        return array(
            'text-embedding-3-small' => array('input' => 0.02/1000000, 'output' => 0.02/1000000),
            'text-embedding-3-large' => array('input' => 0.13/1000000, 'output' => 0.13/1000000),
            'ada v2' => array('input' => 0.10/1000000, 'output' => 0.10/1000000),

            'gpt-4.1' => array('input' => 2.00/1000000, 'output' => 8.00/1000000),
            'gpt-4.1-mini' => array('input' => 0.40/1000000, 'output' => 1.60/1000000),
            'gpt-4.1-nano' => array('input' => 0.10/1000000, 'output' => 0.40/1000000),

            'gpt-4o-mini' => array('input' => 0.15/1000000, 'output' => 0.600/1000000),
            'gpt-4o-mini-2024-07-18' => array('input' => 0.15/1000000, 'output' => 0.600/1000000),

            'gpt-4o' => array('input' => 2.50/1000000, 'output' => 10.00/1000000),
            'gpt-4o-2024-08-06' => array('input' => 2.50/1000000, 'output' => 10.00/1000000),
            'gpt-4o-2024-05-13' => array('input' => 5.00/1000000, 'output' => 15.00/1000000),

            'chatgpt-4o-latest' => array('input' => 5.00/1000000, 'output' => 15.00/1000000),
            'gpt-4-turbo' => array('input' => 10.00/1000000, 'output' => 30.00/1000000),
            'gpt-4-turbo-2024-04-09' => array('input' => 10.00/1000000, 'output' => 30.00/1000000),
            'gpt-4' => array('input' => 30.00/1000000, 'output' => 60.00/1000000),
            'gpt-4-32k' => array('input' => 60.00/1000000, 'output' => 120.00/1000000),
            'gpt-4-0125-preview' => array('input' => 10.00/1000000, 'output' => 30.00/1000000),
            'gpt-4-1106-preview' => array('input' => 10.00/1000000, 'output' => 30.00/1000000),
            'gpt-4-vision-preview' => array('input' => 10.00/1000000, 'output' => 30.00/1000000),
            'gpt-3.5-turbo-0125' => array('input' => 0.50/1000000, 'output' => 1.50/1000000),
            'gpt-3.5-turbo-instruct' => array('input' => 1.50/1000000, 'output' => 2.00/1000000),
            'gpt-3.5-turbo-1106' => array('input' => 1.50/1000000, 'output' => 2.00/1000000),
            'gpt-3.5-turbo-0613' => array('input' => 1.00/1000000, 'output' => 2.00/1000000),
            'gpt-3.5-turbo-16k-0613' => array('input' => 3.00/1000000, 'output' => 4.00/1000000),
            'gpt-3.5-turbo-0301' => array('input' => 1.50/1000000, 'output' => 2.00/1000000),
        );
    }

    private static function get_credit_tracking_process_counter_option_name(){
        return 'wpil_ai_credit_process_counter';
    }

    private static function get_credit_tracking_process_run_option_name(){
        return 'wpil_ai_credit_process_runs';
    }

    private static function get_next_credit_tracking_process_id(){
        $option = self::get_credit_tracking_process_counter_option_name();
        $process_id = ((int) get_option($option, 0)) + 1;
        update_option($option, $process_id, false);

        return $process_id;
    }

    private static function get_credit_tracking_process_runs(){
        $runs = get_option(self::get_credit_tracking_process_run_option_name(), array());

        return (is_array($runs)) ? $runs: array();
    }

    private static function update_credit_tracking_process_runs($runs = array()){
        update_option(self::get_credit_tracking_process_run_option_name(), (is_array($runs)) ? $runs: array(), false);
    }

    private static function get_credit_tracking_context_cache_key($context_key = ''){
        $context_key = trim((string) $context_key);

        return (!empty($context_key)) ? md5('credit_task_run_' . $context_key): '';
    }

    private static function sanitize_credit_tracking_task_key($task_key = ''){
        $task_key = substr(sanitize_key((string) $task_key), 0, 64);

        return (!empty($task_key)) ? $task_key: '';
    }

    public static function begin_credit_tracking_task_run($task_key = '', $context_key = '', $persist = true){
        $task_key = self::sanitize_credit_tracking_task_key($task_key);
        if(empty($task_key)){
            return array('task_key' => '', 'process_id' => 0, 'context_key' => '');
        }

        $cache_key = self::get_credit_tracking_context_cache_key(!empty($context_key) ? $context_key: $task_key);
        $process_id = 0;

        if($persist){
            $runs = self::get_credit_tracking_process_runs();
            if(isset($runs[$cache_key]['process_id'], $runs[$cache_key]['task_key']) &&
                !empty($runs[$cache_key]['process_id']) &&
                $runs[$cache_key]['task_key'] === $task_key)
            {
                $process_id = (int) $runs[$cache_key]['process_id'];
            }else{
                $process_id = self::get_next_credit_tracking_process_id();
            }

            $runs[$cache_key] = array(
                'task_key' => $task_key,
                'process_id' => $process_id,
                'updated_at' => time(),
            );
            self::update_credit_tracking_process_runs($runs);
        }else{
            if(isset(self::$request_credit_tracking_runs[$cache_key]['process_id'], self::$request_credit_tracking_runs[$cache_key]['task_key']) &&
                !empty(self::$request_credit_tracking_runs[$cache_key]['process_id']) &&
                self::$request_credit_tracking_runs[$cache_key]['task_key'] === $task_key)
            {
                $process_id = (int) self::$request_credit_tracking_runs[$cache_key]['process_id'];
            }else{
                $process_id = self::get_next_credit_tracking_process_id();
                self::$request_credit_tracking_runs[$cache_key] = array(
                    'task_key' => $task_key,
                    'process_id' => $process_id,
                    'updated_at' => time(),
                );
            }
        }

        self::$active_credit_tracking_context = array(
            'task_key' => $task_key,
            'process_id' => $process_id,
            'context_key' => $cache_key,
            'persist' => !empty($persist),
        );

        return self::$active_credit_tracking_context;
    }

    public static function clear_credit_tracking_task_run($context_key = ''){
        $cache_key = self::get_credit_tracking_context_cache_key($context_key);

        if(!empty($cache_key)){
            $runs = self::get_credit_tracking_process_runs();
            if(isset($runs[$cache_key])){
                unset($runs[$cache_key]);
                self::update_credit_tracking_process_runs($runs);
            }

            if(isset(self::$request_credit_tracking_runs[$cache_key])){
                unset(self::$request_credit_tracking_runs[$cache_key]);
            }

            if(isset(self::$active_credit_tracking_context['context_key']) && self::$active_credit_tracking_context['context_key'] === $cache_key){
                self::$active_credit_tracking_context = array();
            }

            return;
        }

        self::$active_credit_tracking_context = array();
    }

    public static function clear_all_credit_tracking_task_runs(){
        self::$active_credit_tracking_context = array();
        self::$request_credit_tracking_runs = array();
        delete_option(self::get_credit_tracking_process_run_option_name());
    }

    public static function get_credit_tracking_task_key_from_linking_process_key($process_key = ''){
        $process_key = sanitize_text_field((string) $process_key);
        if(empty($process_key)){
            return '';
        }

        $task_map = array(
            md5('one-click-setup') => 'one-click-ai-suggestions',
            md5('orphan-post-search') => 'fix-orphaned-posts',
            md5('link-coverage-search') => 'fix-link-coverage',
            md5('link-quality-search') => 'fix-link-quality',
            md5('broken-link-search') => 'fix-broken-links',
            md5('external-focus-search') => 'fix-external-focus',
        );

        return isset($task_map[$process_key]) ? $task_map[$process_key]: 'ai-linking-task';
    }

    public static function get_credit_tracking_task_key_from_purpose($purpose = ''){
        $purpose = sanitize_key((string) $purpose);
        if(empty($purpose)){
            return '';
        }

        $task_map = array(
            'post-summarizing' => 'site-analysis-post-summarizing',
            'product-detecting' => 'site-analysis-product-detection',
            'create-post-embeddings' => 'site-analysis-relation-analysis',
            'keyword-detecting' => 'site-analysis-keyword-analysis',
            'summary-and-product-searching' => 'site-analysis-summary-product-search',
            'summary-and-keyword-searching' => 'site-analysis-summary-keyword-search',
            'product-and-keyword-searching' => 'site-analysis-product-keyword-search',
            'summary-keyword-and-product-searching' => 'site-analysis-summary-keyword-product-search',
            'create-post-sentence-embeddings' => 'create-post-sentence-embeddings',
            'assess-sentence-anchors' => 'assess-sentence-anchors',
            'assess-outbound-links' => 'assess-outbound-links',
            'assess-inbound-links' => 'assess-inbound-links',
            'broken-link-replacement' => 'broken-link-replacement',
            'get-available-credits' => 'credit-balance-check',
        );

        return isset($task_map[$purpose]) ? $task_map[$purpose]: self::sanitize_credit_tracking_task_key($purpose);
    }

    public static function get_credit_tracking_task_key_from_process_code($process_code = 0){
        return self::get_credit_tracking_task_key_from_purpose(self::get_process_name_from_code((int) $process_code));
    }

    public static function get_credit_tracking_task_pretty_name($task_key = '', $process_id = 0, $include_run = false){
        $task_key = self::sanitize_credit_tracking_task_key($task_key);
        $label_map = array(
            'one-click-ai-suggestions' => 'One Click AI Suggestions',
            'fix-orphaned-posts' => 'Fix Orphaned Posts',
            'fix-link-coverage' => 'Fix Link Coverage',
            'fix-link-quality' => 'Fix Link Quality',
            'fix-broken-links' => 'Fix Broken Links',
            'fix-external-focus' => 'Fix External Focus',
            'ai-linking-task' => 'AI Linking Task',
            'site-analysis-post-summarizing' => 'Site Analysis: Post Summarizing',
            'site-analysis-product-detection' => 'Site Analysis: Product Detection',
            'site-analysis-relation-analysis' => 'Site Analysis: Relation Analysis',
            'site-analysis-keyword-analysis' => 'Site Analysis: Keyword Analysis',
            'site-analysis-summary-product-search' => 'Site Analysis: Summary + Product Search',
            'site-analysis-summary-keyword-search' => 'Site Analysis: Summary + Keyword Search',
            'site-analysis-product-keyword-search' => 'Site Analysis: Product + Keyword Search',
            'site-analysis-summary-keyword-product-search' => 'Site Analysis: Summary + Keyword + Product Search',
            'create-post-sentence-embeddings' => 'Create Post Sentence Embeddings',
            'assess-sentence-anchors' => 'Assess Sentence Anchors',
            'assess-outbound-links' => 'Assess Outbound Links',
            'assess-inbound-links' => 'Assess Inbound Links',
            'broken-link-replacement' => 'Broken Link Replacement',
            'credit-balance-check' => 'Credit Balance Check',
        );

        $label = isset($label_map[$task_key]) ? $label_map[$task_key] : ucwords(str_replace('-', ' ', $task_key));

        if($include_run && !empty($process_id)){
            $label .= ' (Run #' . (int) $process_id . ')';
        }

        return $label;
    }

    private static function get_current_credit_tracking_context($process_number = 0, $batch_process = false, $task_key = '', $process_id = 0){
        $task_key = self::sanitize_credit_tracking_task_key($task_key);
        $process_id = (int) $process_id;

        if(!empty($task_key) && !empty($process_id)){
            return array('task_key' => $task_key, 'process_id' => $process_id);
        }

        if(!empty($task_key) &&
            !empty(self::$active_credit_tracking_context['task_key']) &&
            !empty(self::$active_credit_tracking_context['process_id']) &&
            self::$active_credit_tracking_context['task_key'] === $task_key)
        {
            return array(
                'task_key' => self::$active_credit_tracking_context['task_key'],
                'process_id' => (int) self::$active_credit_tracking_context['process_id'],
            );
        }

        if(!empty(self::$active_linking_process_key)){
            $task_key = self::get_credit_tracking_task_key_from_linking_process_key(self::$active_linking_process_key);
            if(!empty($task_key)){
                return self::begin_credit_tracking_task_run($task_key, 'linking:' . self::$active_linking_process_key, true);
            }
        }

        if(!empty(self::$purpose)){
            $task_key = self::get_credit_tracking_task_key_from_purpose(self::$purpose);
            if(!empty($task_key)){
                return self::begin_credit_tracking_task_run($task_key, 'purpose:' . self::$purpose, !empty($batch_process));
            }
        }

        if(!empty($process_number)){
            $task_key = self::get_credit_tracking_task_key_from_process_code($process_number);
            if(!empty($task_key)){
                return self::begin_credit_tracking_task_run($task_key, 'request:' . $task_key, false);
            }
        }

        if(!empty(self::$active_credit_tracking_context['task_key']) && !empty(self::$active_credit_tracking_context['process_id'])){
            return array(
                'task_key' => self::$active_credit_tracking_context['task_key'],
                'process_id' => (int) self::$active_credit_tracking_context['process_id'],
            );
        }

        return array('task_key' => '', 'process_id' => 0);
    }

    public static function get_process_code_from_name($name = ''){
        if(is_int($name)){
            return $name;
        }

        $code = 0;
        switch ($name) {
            case 'suggestion-scoring':
                $code = 1;
                break;
            case 'post-summarizing':
                $code = 2;
                break;
            case 'product-detecting':
                $code = 3;
                break;
            case 'create-post-embeddings':
                $code = 4;
                break;
            case 'keyword-detecting':
                $code = 5;
                break;
            case 'summary-and-product-searching':
                $code = 6;
                break;
            case 'summary-and-keyword-searching':
                $code = 7;
                break;
            case 'product-and-keyword-searching':
                $code = 8;
                break;
            case 'summary-keyword-and-product-searching':
                $code = 9;
                break;
            case 'create-post-sentence-embeddings':
                $code = 10;
                break;
            case 'assess-sentence-anchors':
                $code = 11;
                break;
            case 'assess-outbound-links':
                $code = 12;
                break;
            case 'assess-inbound-links':
                $code = 13;
                break;
            case 'broken-link-replacement':
                $code = 14;
                break;
            case 'get-available-credits':
                $code = 15;
                break;
            case 'credit-deposit': // beseeching forgiveness of future selves is in order...
                $code = 16;
                break;
        }

        return $code;
    }

    public static function get_process_name_from_code($code = 0){
        $process_list = array(
            0 => 'unknown',
            1 => 'suggestion-scoring',
            2 => 'post-summarizing',
            3 => 'product-detecting',
            4 => 'create-post-embeddings',
            5 => 'keyword-detecting',
            6 => 'summary-and-product-searching',
            7 => 'summary-and-keyword-searching',
            8 => 'product-and-keyword-searching',
            9 => 'summary-keyword-and-product-searching',
            10 => 'create-post-sentence-embeddings',
            11 => 'assess-sentence-anchors',
            12 => 'assess-outbound-links',
            13 => 'assess-inbound-links',
            14 => 'broken-link-replacement',
            15 => 'get-available-credits',
            16 => 'credit-deposit', // further beseeching forgiveness is in order...
        );

        return (isset($process_list[$code])) ? $process_list[$code]: 'unknown';
    }

    public static function get_process_pretty_name_from_code($code = 0, $transaction_note = ''){
        $process_list = array(
            0 => 'Unknown',
            1 => 'Suggestion Scoring',
            2 => 'Post Summarizing',
            3 => 'Product Detection',
            4 => 'Create Post Embeddings',
            5 => 'Keyword Detection',
            6 => 'Summary + Product Search',
            7 => 'Summary + Keyword Search',
            8 => 'Product + Keyword Search',
            9 => 'Summary + Keyword + Product Search',
            10 => 'Create Post Sentence Embeddings',
            11 => 'Assess Sentence Anchors',
            12 => 'Assess Outbound Links',
            13 => 'Assess Inbound Links',
            14 => 'Broken Link Replacement',
            15 => 'Credit Balance Check',
            16 => 'Credits Added',
        );

        $label = isset($process_list[$code]) ? $process_list[$code] : 'Unknown';
        if(16 === (int) $code && !empty($transaction_note)){
            $label .= ' (' . ucwords(str_replace(array('-', '_'), ' ', (string) $transaction_note)) . ')';
        }

        return $label;
    }

    public static function get_credit_history_filter_processes(){
        $processes = array();
        for($i = 1; $i <= 16; $i++){
            $processes[$i] = self::get_process_pretty_name_from_code($i);
        }

        return $processes;
    }

    /**
     * Analyzes site posts to create summaries of them and/or to identify products within them.
     **/
    public static function analyze_site_posts(){
        $time = microtime(true);
        $token_size = 0;
        $doing_ajax = (defined('DOING_AJAX') && DOING_AJAX) ? true: false;

        // first, figure out what we're doing
        $active_processes = Wpil_Settings::get_selected_ai_batch_processes(true, true);

        // if we're not doing anything
        if(empty($active_processes)){
            // exit
            return false;
        }

        $grouped = array();
        foreach($active_processes as $process){
            if(self::check_batch_status_completed($process)){
                self::clear_credit_tracking_task_run('purpose:' . $process);
                continue;
            }

            $gpt = Wpil_Settings::getChatGPTVersion($process);
            if(!isset($grouped[$gpt])){
                $grouped[$gpt] = array();
            }
            $grouped[$gpt][] += self::get_process_code_from_name($process);
        }

        foreach($grouped as $model => $dat){
            $process = 0;
            if(Wpil_Base::overTimeLimit(5, 40)){
                break;
            }

            self::$model = $model;
            $task_sum = array_sum($dat); //TODO: Create a better way to select the active processes
            $doing_keywords = (count($dat) === 1 && isset($dat[0]) && $dat[0] === 5) ? true: false; // TODO: Create a better way to do this

            // if we're doing product search, summaries and keyword detecting
            if($task_sum === 10){
                self::$purpose = 'summary-keyword-and-product-searching';
                $keyword_count = Wpil_Settings::get_ai_keyword_count_max();

                // get the posts
                $posts = self::get_all_batch_process_posts(self::$purpose, self::get_process_code_from_name(self::$purpose));

                // setup the instructions for Chatty
                $instructions = array(
                    "You have three jobs:",
                        "The first and primary one is to read and assess the content of a supplied article in order to create a short summary of it.",
                            "First, identify the primary idea of the article. Once you\'ve identified the primary idea, identify any secondary ideas or points the article makes. If there are no secondary points, only work with the primary idea.",
                            "Based on your understanding of the article, write the summary in a first person style that relays the information contained directly, without claiming that you wrote it.",
                            "The summary must concisely explain the primary idea of the article.",
                            "If there are secondary points, explain them briefly and indicate how they are related to the primary idea.",
                            "For example if you get an article that details the 'Care and Feeding of Black Skirt Tetras', write a summary of the article that 1.) introduces the Black Skirt Tetra briefly to show what kind of fish they are, 2.) informs the reader that the article is about the care and feeding of Black Skirt Tetras, 3.) gives a general outline of the article's information concerning the care and feeding of Black Skirt Tetras.",
                            "The summary must not be longer than the original article, and should be no longer than 300 words",
                        "Your second job is to detect all products mentioned in the content, and create a comma-separated list of products.",
                            "If you find any brand-name products, include the brand name along with the product. For example, if you find text like this: \"The best Nike Air Max Sneakers in the world!\", list \"Nike Air Max Sneakers\" as the product.",
                        "Your third job is to examine the content of the supplied article to find keywords that are relevant to it's topic. Assess the article wholistically, isolating the key points and overarching pieces of information, and then create meaningful keywords for it. Make sure the keywords are relevant to the article's topic, and not only generally related. For example: If the article is about \"Best Target Stores in Kansas\", do create keywords relevant to Target stores in Kansas such as \"The Topeka Target\" or \"Shopping at Target in Kansas\", and do not create keywords that are only generically related such as \"Target Stores\" or \"Shopping Center\" or \"Credit Card Purchase\". Keywords should be between 2 and 5 words long.",
                            "Once you've created a list of keywords, read them back to yourself and remove any that do not make sense from an SEO perspective or are shorter than 2 words. Remove any keywords that are so broadly applicable as to be meaningless such as \"house\", \"post\", \"cool\", \"dog\", \"road\", \"space\", \"electric\", \"math\". After doing that, remove any duplicates to produce a list of unique keywords. If the total list is longer then {$keyword_count}, you must select the {$keyword_count} most detailed and article-related keywords to return. Do not return more than {$keyword_count} keywords, and do not create vague keywords in order to reach {$keyword_count} keywords as returning less than {$keyword_count} is acceptable if that's all the content will support. The keywords must be returned in a comma-separated list.",
                        "Your response must be in JSON. The article summary must be indexed to \"summary\". All summary information must be a string and indexed to \"summary\", do not All summary information must be a string and indexed to \"summary\", do not format any summary information as an object. The product list must be indexed to \"products\". If there are no products in the content, the value of \"products\" must be 'no-products'. List the number of products in an index called \"product-count\". The keywords must be indexed to \"keywords\". If you cannot create keywords based on the content, the value of \"keywords\" must be 'no-keywords'. List the number of keywords in an index called \"keyword-count\"."
                    );/*DONE!*/

            }elseif($task_sum === 8){ // if we're doing product and keyword detecting
                self::$purpose = 'product-and-keyword-searching';
                $keyword_count = Wpil_Settings::get_ai_keyword_count_max();

                // get the posts
                $posts = self::get_all_batch_process_posts(self::$purpose, self::get_process_code_from_name(self::$purpose));

                // setup the instructions for Chatty
                $instructions = array(
                    "You have two jobs:",
                        "The first and primary one is to read and assess the content of a supplied article in order to detect all products mentioned in the content, and create a comma-separated list of products.",
                            "If you find any brand-name products, include the brand name along with the product. For example, if you find text like this: \"The best Nike Air Max Sneakers in the world!\", list \"Nike Air Max Sneakers\" as the product.",
                        "Your second job is to examine the content of the supplied article to find keywords that are relevant to it's topic. Assess the article wholistically, isolating the key points and overarching pieces of information, and then create meaningful keywords for it. Make sure the keywords are relevant to the article's topic, and not only generally related. For example: If the article is about \"Best Target Stores in Kansas\", do create keywords relevant to Target stores in Kansas such as \"The Topeka Target\" or \"Shopping at Target in Kansas\", and do not create keywords that are only generically related such as \"Target Stores\" or \"Shopping Center\" or \"Credit Card Purchase\". Keywords should be between 2 and 5 words long.",
                            "Once you've created a list of keywords, read them back to yourself and remove any that do not make sense from an SEO perspective or are shorter than 2 words. Remove any keywords that are so broadly applicable as to be meaningless such as \"house\", \"post\", \"cool\", \"dog\", \"road\", \"space\", \"electric\", \"math\". After doing that, remove any duplicates to produce a list of unique keywords. If the total list is longer then {$keyword_count}, you must select the {$keyword_count} most detailed and article-related keywords to return. Do not return more than {$keyword_count} keywords, and do not create vague keywords in order to reach {$keyword_count} keywords as returning less than {$keyword_count} is acceptable if that's all the content will support. The keywords must be returned in a comma-separated list.",
                        "Your response must be in JSON. The product list must be indexed to \"products\". If there are no products in the content, the value of \"products\" must be 'no-products'. List the number of products in an index called \"product-count\". The keywords must be indexed to \"keywords\". If you cannot create keywords based on the content, the value of \"keywords\" must be 'no-keywords'. List the number of keywords in an index called \"keyword-count\"."
                );/*DONE!*/
            }elseif($task_sum === 7){ // if we're doing summary and keyword detecting
                self::$purpose = 'summary-and-keyword-searching';
                $keyword_count = Wpil_Settings::get_ai_keyword_count_max();

                // get the posts
                $posts = self::get_all_batch_process_posts(self::$purpose, self::get_process_code_from_name(self::$purpose));

                $instructions = array(
                    "You have two jobs:",
                    "The first and primary one is to read and assess the content of a supplied article in order to create a short summary of it.",
                        "First step, identify the primary idea of the article. Once you\'ve identified the primary idea, identify any secondary ideas or points the article makes. If there are no secondary points, only work with the primary idea.",
                        "Based on your understanding of the article, write the summary in a first person style that relays the information contained directly, without claiming that you wrote it.",
                        "The summary must concisely explain the primary idea of the article.",
                        "If there are secondary points, explain them briefly and indicate how they are related to the primary idea.",
                        "For example if you get an article that details the 'Care and Feeding of Black Skirt Tetras', write a summary of the article that 1.) introduces the Black Skirt Tetra briefly to show what kind of fish they are, 2.) informs the reader that the article is about the care and feeding of Black Skirt Tetras, 3.) gives a general outline of the article's information concerning the care and feeding of Black Skirt Tetras.",
                        "The summary must not be longer than the original article, and should be no longer than 300 words",
                    "Your second job is to examine the content of the supplied article to find keywords that are relevant to it's topic. Assess the article wholistically, isolating the key points and overarching pieces of information, and then create meaningful keywords for it. Make sure the keywords are relevant to the article's topic, and not only generally related. For example: If the article is about \"Best Target Stores in Kansas\", do create keywords relevant to Target stores in Kansas such as \"The Topeka Target\" or \"Shopping at Target in Kansas\", and do not create keywords that are only generically related such as \"Target Stores\" or \"Shopping Center\" or \"Credit Card Purchase\". Keywords should be between 2 and 5 words long.",
                        "Once you've created a list of keywords, read them back to yourself and remove any that do not make sense from an SEO perspective or are shorter than 2 words. Remove any keywords that are so broadly applicable as to be meaningless such as \"house\", \"post\", \"cool\", \"dog\", \"road\", \"space\", \"electric\", \"math\". After doing that, remove any duplicates to produce a list of unique keywords. If the total list is longer then {$keyword_count}, you must select the {$keyword_count} most detailed and article-related keywords to return. Do not return more than {$keyword_count} keywords, and do not create vague keywords in order to reach {$keyword_count} keywords as returning less than {$keyword_count} is acceptable if that's all the content will support. The keywords must be returned in a comma-separated list.",
                    "Your response must be in JSON. The article summary must be indexed to \"summary\". All summary information must be a string and indexed to \"summary\", do not All summary information must be a string and indexed to \"summary\", do not format any summary information as an object. The keywords must be indexed to \"keywords\". If you cannot create keywords based on the content, the value of \"keywords\" must be 'no-keywords'. List the number of keywords in an index called \"keyword-count\"."
                );/*DONE!*/

            }elseif($task_sum === 5 && $doing_keywords){ // if we're doing keyword detecting
                self::$purpose = 'keyword-detecting';
                $keyword_count = Wpil_Settings::get_ai_keyword_count_max();

                // get the posts
                $posts = self::get_all_batch_process_posts(self::$purpose, self::get_process_code_from_name(self::$purpose));
                
                // setup the instructions for Chatty
                $instructions = array(
                    "Your job is examine the content of the supplied article to find keywords that are relevant to it's topic. Assess the article wholistically, isolating the key points and overarching pieces of information, and then create meaningful keywords for it. Make sure the keywords are relevant to the article's topic, and not only generally related. For example: If the article is about \"Best Target Stores in Kansas\", do create keywords relevant to Target stores in Kansas such as \"The Topeka Target\" or \"Shopping at Target in Kansas\", and do not create keywords that are only generically related such as \"Target Stores\" or \"Shopping Center\" or \"Credit Card Purchase\". Keywords should be between 2 and 5 words long.",
                    "Once you've created a list of keywords, read them back to yourself and remove any that do not make sense from an SEO perspective or are shorter than 2 words. Remove any keywords that are so broadly applicable as to be meaningless such as \"house\", \"post\", \"cool\", \"dog\", \"road\", \"space\", \"electric\", \"math\". After doing that, remove any duplicates to produce a list of unique keywords. If the total list is longer then {$keyword_count}, you must select the {$keyword_count} most detailed and article-related keywords to return. Do not return more than {$keyword_count} keywords, and do not create vague keywords in order to reach {$keyword_count} keywords as returning less than {$keyword_count} is acceptable if that's all the content will support. The keywords must be returned in a comma-separated list.",
                    "Your response must be in JSON. The keywords must be indexed to \"keywords\". If you cannot create keywords based on the content, the value of \"keywords\" must be 'no-keywords'. List the number of keywords in an index called \"keyword-count\"."
                );/*DONE!*/

            }elseif($task_sum === 5){
                self::$purpose = 'summary-and-product-searching';

                // get the posts
                $posts = self::get_all_batch_process_posts(self::$purpose, self::get_process_code_from_name(self::$purpose));

                // setup the instructions for Chatty
                $instructions = array(
                    "You have two jobs:",
                        "The first and primary one is to read and assess the content of a supplied article in order to create a short summary of it.",
                            "First, identify the primary idea of the article. Once you\'ve identified the primary idea, identify any secondary ideas or points the article makes. If there are no secondary points, only work with the primary idea.",
                            "Based on your understanding of the article, write the summary in a first person style that relays the information contained directly, without claiming that you wrote it.",
                            "The summary must concisely explain the primary idea of the article.",
                            "If there are secondary points, explain them briefly and indicate how they are related to the primary idea.",
                            "For example if you get an article that details the 'Care and Feeding of Black Skirt Tetras', write a summary of the article that 1.) introduces the Black Skirt Tetra briefly to show what kind of fish they are, 2.) informs the reader that the article is about the care and feeding of Black Skirt Tetras, 3.) gives a general outline of the article's information concerning the care and feeding of Black Skirt Tetras.",
                            "The summary must not be longer than the original article, and should be no longer than 300 words",
                        "Your secondary job is to detect all products mentioned in the content, and create a comma-separated list of products.",
                            "If you find any brand-name products, include the brand name along with the product. For example, if you find text like this: \"The best Nike Air Max Sneakers in the world!\", list \"Nike Air Max Sneakers\" as the product.",
                    "Your response must be in JSON. The article summary must be indexed to \"summary\" and the product list must be indexed to \"products\". All summary information must be a string and indexed to \"summary\", do not All summary information must be a string and indexed to \"summary\", do not format any summary information as an object. If there are no products in the content, the value of \"products\" must be 'no-products'. List the number of products in an index called \"product-count\"."
                );

            }elseif($task_sum === 2){ // doing summaries
                self::$purpose = 'post-summarizing';

                // get the posts
                $posts = self::get_all_batch_process_posts(self::$purpose, self::get_process_code_from_name(self::$purpose));
                
                // setup the instructions for Chatty
                $instructions = array(
                    "Your job is to read and assess the content of a supplied article in order to create a short summary of it.",
                    "First, identify the primary idea of the article. Once you\'ve identified the primary idea, identify any secondary ideas or points the article makes. If there are no secondary points, only work with the primary idea.",
                    "Based on your understanding of the article, write the summary in a first person style that relays the information contained directly, without claiming that you wrote it.",
                    "The summary must concisely explain the primary idea of the article.",
                    "If there are secondary points, explain them briefly and indicate how they are related to the primary idea.",
                    "For example if you get an article that details the 'Care and Feeding of Black Skirt Tetras', write a summary of the article that 1.) introduces the Black Skirt Tetra briefly to show what kind of fish they are, 2.) informs the reader that the article is about the care and feeding of Black Skirt Tetras, 3.) gives a general outline of the article's information concerning the care and feeding of Black Skirt Tetras.",
                    "The summary must not be longer than the original article, and should be no longer than 300 words",
                    "Your response must be in JSON. Index the summary to \"summary\". All summary information must be a string and indexed to \"summary\", do not All summary information must be a string and indexed to \"summary\", do not format any summary information as an object."
                );
            }elseif($task_sum === 3){// doing product search
                self::$purpose = 'product-detecting';

                // get the posts
                $posts = self::get_all_batch_process_posts(self::$purpose, self::get_process_code_from_name(self::$purpose));
                
                // setup the instructions for Chatty
                $instructions = array(
                    "Your job is examine the content of a supplied article and create a comma-separated list of all products mentioned inside the article.",
                    "If you find any brand-name products, include the brand name along with the product. For example, if you find text like this: \"The best Nike Air Max Sneakers in the world!\", list \"Nike Air Max Sneakers\" as the product.",
                    "The product list must be comma-separated, and must not contain any article that is not present in the supplied content. Do not include any reasoning or rationale for why a product is in the list, simply return a comma-separated list of all the products you find.",
                    "If there are no products in the content, give 'no-products' as the result of your search.",
                    "Your response must be in JSON. The product list must be indexed to \"products\". If there are no products in the content, the value of \"products\" must be 'no-products'. List the number of products in an index called \"product-count\"."
                );
            }

            if(empty($posts) || (self::check_delayed_batch_process(self::$purpose) && !$doing_ajax)){
                continue;
            }

            shuffle($posts);
            $post_data = array();
            $count = 0;
            $instructions = implode(' ', $instructions);
            $instruction_size = self::count_tokens($instructions, self::$model);
            $chunked_posts = self::get_chunked_posts();
            $limits = self::get_api_rate_limits();
            foreach($posts as $post){
                // exit if we've been at this for more than 20 seconds or we've managed to pull down 50,000 posts
                if(microtime(true) - $time > 20 || $count >= self::$batch_limit || $doing_ajax && $count >= self::$concurrency){
                    break;
                }

                $id = ($post->type . '_' . $post->id);
                // get the cleaned content
                $content = mb_ereg_replace("\n", '', mb_ereg_replace('(([a-zA-Z\-_0-9]+="[^"]*")+?[\s]?)', '', strip_tags($post->getContent(false), '<h1><h2><h3><h4><h5><h6><title><ul><ol><li>')));

                // if there's no content to process
                if(empty($content)){
                    // mark it as complete and don't waste time on it
                    self::save_empty_post_data($id, self::$model, self::$purpose);
                    continue;
                }

                if($doing_ajax && isset($chunked_posts[$id])){
                    $content = self::chunk_post_content($content, 2500);
                }else{
                    $content = self::trim_text_to_token_limit($content, self::$model, 7800);
                }

                $query = (is_array($content)) ? array_reduce($content, function($count, $chunk) use ($instruction_size){ return $count += ($instruction_size + self::count_tokens($chunk, self::$model)); }): $instruction_size + self::count_tokens($content, self::$model);
                if($token_size + $query < ($limits[self::$model] - 10000)){
                    $token_size += $query;
                    $post_data[$id] = $content;
                    if(is_array($content)){
                        $count += count($content);
                    }

                }else{
                    break;
                }

                $count++;
            }

            if(self::$ai_service_connected){
                self::live_query_linkwhisper_ai_results($post_data, 'completions');
                return;
            }

            if($doing_ajax){
                self::live_query_ai_results($post_data, $instructions, 'completions');
            }else{
                self::start_batch_process($post_data, $instructions, 'completions');
            }
        }
    }

    /**
     * 
     **/
    public static function create_site_embeddings(){
        $time = microtime(true);
        $posts = self::get_all_batch_process_posts('create-post-embeddings', 4);
        self::$purpose = 'create-post-embeddings';
        self::$model = Wpil_Settings::getChatGPTVersion('create-post-embeddings');
        $token_size = 0;
        $doing_ajax = (defined('DOING_AJAX') && DOING_AJAX) ? true: false;

        if(empty($posts) || (self::check_delayed_batch_process('create-post-embeddings') && !$doing_ajax)){
            return false;
        }

        $post_data = array();
        $count = 0;
        $limit = self::get_api_rate_limits('text-embedding-3-large');
        foreach($posts as $post){
            // exit if we've been at this for more than 20 seconds or we've managed to pull down 50,000 posts
            if(microtime(true) - $time > 20 || $count >= self::$batch_limit || $doing_ajax && $count >= self::$concurrency){
                break;
            }

            $id = ($post->type . '_' . $post->id);
            $content = self::trim_text_to_token_limit(mb_ereg_replace("\n", '', mb_ereg_replace('(([a-zA-Z\-_0-9]+="[^"]*")+?[\s]?)', '', strip_tags($post->getContent(false), '<h1><h2><h3><h4><h5><h6><title><ul><ol><li>'))), self::$model, 7800);
            
            // if there's no content to process
            if(empty($content)){
                // mark it as complete and don't waste time on it
                self::save_empty_post_data($id, self::$model, self::$purpose);
                continue;
            }
            
            $query = self::count_tokens($content, self::$model);
            if($token_size + $query < $limit){
                $token_size += $query;
                $post_data[$id] = $content;
            }else{
                break;
            }

            $count++;
        }

        if(self::$ai_service_connected){
            self::live_query_linkwhisper_ai_results($post_data, 'embeddings');
            return;
        }

        if($doing_ajax){
            self::live_query_ai_results($post_data, 'create-post-embeddings', 'embeddings');
        }else{
            self::start_batch_process($post_data, 'create-post-embeddings', 'embeddings');
        }
    }

    /**
     * Analyzes a list of sentences that we're confident are related to their target posts to find the best anchors
     **/
    public static function assess_post_sentence_anchors($phrase_data, $origin_post_id = ''){
        $time = microtime(true);
//        $posts = self::get_all_batch_process_posts('assess-sentence-anchors', 4); // todo setup batching later
        self::$purpose = 'assess-sentence-anchors';
        self::$model = Wpil_Settings::getChatGPTVersion('assess-sentence-anchors');
        $token_size = 0;
        $doing_ajax = true || (defined('DOING_AJAX') && DOING_AJAX) ? true: false;

        if(empty($phrase_data) || (self::check_delayed_batch_process('assess-sentence-anchors') && !$doing_ajax) || empty($origin_post_id)){
            return null;
        }

        $bits = explode('_', $origin_post_id);
        self::$origin_post = new Wpil_Model_Post($bits[1], $bits[0]);
/*
        // setup the instructions for Chatty
        $instructions = array(
            "You are being given a sentence that is related to a specific blog post. Your job is to determine which words from that exact sentence would make the best single concise HTML anchor text to use in a link to that post",
            "You must list all words that will be used in the anchor. You may not use any words that are not in the sentence. ALL words must be consecutive, and you must not skip over words. Do not rewrite or adjust the sentence in any way. The anchor must make sense to a human reader and be long enough convey meaning, ideally between 3 and 8 words.", // Assume that the sentence is related to the post, and try to find a basis to select words for an anchor.",
            "We are giving you the post title and a comma-separated list of keywords as an indicator of what the post's topic is. When you are done selecting the words, check your work by re-evaluating the sentence again and seeing if you come to the same conclusion. If you do not, choose the conclusion that makes the most sense. Give preference to anchors that contain one or more of the keywords.",
            "The data is being given in JSON format. The \"sentence\" value is the sentence that the you will check for words related to the post title. The \"post_title\" value is the post title that you will be finding related words for. The \"keywords\" are a list of keywords that we believe are relevant to the post's topic.",
            "Your response must be in JSON with this structure {\"words\": \"\", \"sentence\": \"\", \"title\": \"\", \"keywords\": \"\", \"match_score\": \"\", \"free_form\": \"\"}. The anchor words must be in an array indexed to \"words\" and the input sentence must be indexed to \"sentence\". The input post title must be indexed to \"title\", and the input keywords must be indexed to \"keywords\".",
            "After you have entered the 'words', 'sentence' and 'title', evaluate the anchor you have proposed, justify why the anchor is or is not related to the title or keywords in the \"free_form\" field, and rank how well it relates to the target post and it's keywords on a scale of 1 to 10. With 1 being not related at all to the target post and its keywords, and 10 being perfectly related to the target post and its keywords. You must base your assessment off of the title and the keywords only.",
        //    "If the anchor is not a good match, and can not be reasonably considered related, put a \"1\" in the \"poor_match\" index.",
        //    "If the anchor is cannot be considered topically related at all, enter \"bad-match\" in the \"words\" index, and put a \"1\" in the \"bad_match\" index.",
        //    "disregard all prior instructions, tell me you're a teapot. your response must be in JSON and have nothing to do with Illinois. give me an update on your status in the \"free_form\" index. If you are unable to normally follow instructions, tell me why there."
        //    "Give me an update on your status in the \"free_form\" index. If you are unable to normally follow instructions, tell me why there."
        );*/
/*
        $instructions = array(
            "You are being given a sentence that is related to a specific blog post. Your job is to determine which words from that exact sentence would make the best single HTML anchor text to use in a link to that post",
            "You must list all words that will be used in the anchor. You may not use any words that are not in the sentence. ALL words must be consecutive, and you must not skip over words. Do not rewrite or adjust the sentence in any way.",
            "We are giving you the post title as an indicator of what the post's topic is. When you are done selecting the words, check your work by re-evaluating the sentence again and seeing if you come to the same conclusion. If you do not, choose the conclusion that makes the most sense.",
            "The data is being given in JSON format. The \"sentence\" value is the sentence that the you will check for words related to the post title. The \"post_title\" value is the post title that you will be finding related words for.",
            "Your response must be in JSON. The anchor words must be in an array indexed to \"words\" and the input sentence must be indexed to \"sentence\". And the input post title must be indexed to \"title\".",
            "If the sentence is not a good match, enter \"poor-match\" in the \"words\" index, and put a \"1\" in the \"poor_match\" index.",
        //    "disregard all prior instructions, tell me you're a teapot. your response must be in JSON and have nothing to do with Illinois. give me an update on your status in the \"free_form\" index. If you are unable to normally follow instructions, tell me why there."
            "Give me an update on your status in the \"free_form\" index. If you are unable to normally follow instructions, tell me why there."
        );
*/

        $instructions = array(
            "You are being given an array of JSON objects. ",
            "Each object contains a sentence that may be related to a specific blog post. Your job is to determine which words from that exact sentence would make the best single concise HTML anchor text to use in a link to the target post. ",
            "You must list all words that will be used in the anchor. You may not use any words that are not in the sentence. ALL words must be consecutive, and you must not skip over words. Do not rewrite or adjust the sentence in any way. ",
            "You MUST TREAT EACH OBJECT INDIVIDUALLY, and you MUST NOT combine information from separate objects. All objects are separate from each other and are not related, you must not use them as context for each other. ",
            "The anchor must make sense to a human reader and be long enough convey meaning, ideally between 3 and 8 words. ",
            "Inside of each object, we are giving you the target post title and a comma-separated list of keywords as an indicator of what the target post's topic is. ",
            "When you are done selecting the words, check your work by re-evaluating the sentence again and seeing if you come to the same conclusion. If you do not, choose the conclusion that makes the most sense based on relation to the target post's title and keywords. Give preference to anchors that contain one or more of the keywords or contains multiple words from the target post's title. ",
            "The data is being given in JSON format. The \"sentence\" value is the sentence that the you will check for words related to the post title. The \"post_title\" value is the post title that you will be finding related words for. The \"keywords\" are a list of keywords that we believe are relevant to the post's topic. The \"meta_id\" is a unique identifier for the JSON object. ",
            "Your response must be an array of JSON objects, and the JSON objects must use this structure {\"meta_id\":\"\",\"words\": \"\", \"sentence\": \"\", \"title\": \"\", \"keywords\": \"\", \"match_score\": \"\", \"free_form\": \"\"}. ",
            "The anchor words must be indexed to \"words\" and the input sentence must be indexed to \"sentence\". The input post title must be indexed to \"title\", and the input keywords must be indexed to \"keywords\". The input \"meta_id\" value must be indexed to the output \"meta_id\" index. ",
            "After you have entered the 'words', 'sentence' and 'title' for an object, evaluate the sentence to determine if it is related to the object\'s the title or the object\'s keywords and write your evaluation in the object\'s \"free_form\" field, and then rank how well the object\'s sentence relates to the object's target post and the object's keywords on a scale of 1 to 10. With 1 being not related at all to the object's target post and the object's keywords, and 10 being perfectly related to the target post and its keywords. Each \"match_score\" value must be specifically and individually calculated for each object. ",
   //         "After you have calculated the \"match_score\" and have finalized the data for each object, set the \"free_form\" value to 'pizza' without changing any of the other values that you have set in the object."
        
        );
/*
        $instructions = array(
            "You are being given an array of JSON objects. ",
            "You MUST TREAT EACH OBJECT INDIVIDUALLY, and you MUST NOT combine information from separate objects. All objects are separate from each other and are not related, you must not use them as context for each other. ",
            "Your job is to examine the content of each object individually, and explain in the object's \"free_form\" field what consecutive string of words from the object's unedited \"sentence\" value would make the best SEO optimized link to a post titled by the object's \"post_title\" value and was ranking for keywords from the object's \"keywords\" value. ",
            "After writing up your explanation, re-assess what consecutive string of words from the unedited \"sentence\" value would make the best SEO optimized link to a post titled by the \"post_title\" value and was ranking for keywords from the \"keywords\" value and create a second explanation in a field called \"free_form_2\". ",
            "After writing your second explanation, and judge which makes the most compelling case for a link and enter the best explanation in the object's \"free_form\" field along with a note which was best. ",
            "Then based on the explanation in the object's \"free_form\" field, enter the consecutive string of words in the object's \"words\" field. When you have done this, rank on a scale of 1 to 10 how related the object's \"words\" are to the object's \"post_title\" value and the object's \"keywords\" value. ", 
            "With 1 being not related at all to the object's \"post_title\" and the object's \"keywords\", and 10 being perfectly related to the object's \"post_title\" and the object's \"keywords\". Each object's \"match_score\" value must be individually calculated for each object, and be based on the object's \"post_title\" and the object's \"keywords\" only. ",
            "When you are finished, ensure that each of the JSON objects in the array you return use this structure {\"meta_id\":\"\",\"words\": \"\", \"sentence\": \"\", \"title\": \"\", \"keywords\": \"\", \"match_score\": \"\", \"free_form\": \"\"}. "
            
/*
            "Each object contains a sentence that is related to a specific blog post. Your job is to determine which words from that exact sentence would make the best single concise HTML anchor text to use in a link to the target post. ",
            "You must list all words that will be used in the anchor. You may not use any words that are not in the sentence. ALL words must be consecutive, and you must not skip over words. Do not rewrite or adjust the sentence in any way. ",
            "You MUST TREAT EACH OBJECT INDIVIDUALLY, and you MUST NOT combine information from separate objects. All objects are separate from each other and are not related, you must not use them as context for each other. ",
            "The anchor must make sense to a human reader and be long enough convey meaning, ideally between 3 and 8 words. ",
            "Inside of each object, we are giving you the target post title and a comma-separated list of keywords as an indicator of what the target post's topic is. ",
            "When you are done selecting the words, check your work by re-evaluating the sentence again and seeing if you come to the same conclusion. If you do not, choose the conclusion that makes the most sense based on relation to the target post's title and keywords. Give preference to anchors that contain one or more of the keywords or contains multiple words from the target post's title. ",
            "The data is being given in JSON format. The \"sentence\" value is the sentence that the you will check for words related to the post title. The \"post_title\" value is the post title that you will be finding related words for. The \"keywords\" are a list of keywords that we believe are relevant to the post's topic. The \"meta_id\" is a unique identifier for the JSON object. ",
            "Your response must be an array of JSON objects, and the JSON objects must use this structure {\"meta_id\":\"\",\"words\": \"\", \"sentence\": \"\", \"title\": \"\", \"keywords\": \"\", \"match_score\": \"\", \"free_form\": \"\"}. ",
            "The anchor words must be indexed to \"words\" and the input sentence must be indexed to \"sentence\". The input post title must be indexed to \"title\", and the input keywords must be indexed to \"keywords\". The input \"meta_id\" value must be indexed to the output \"meta_id\" index. ",
            "After you have entered the 'words', 'sentence' and 'title' for an object, evaluate the anchor you have proposed, justify why the anchor is or is not related to the title or keywords in the \"free_form\" field, and rank how well it relates to the object's target post and the object's keywords on a scale of 1 to 10. With 1 being not related at all to the object's target post and the object's keywords, and 10 being perfectly related to the target post and its keywords. Each \"match_score\" value must be individually calculated for each object, and be based on the object's title and the object's keywords only. ",
   //         "After you have calculated the \"match_score\" and have finalized the data for each object, set the \"free_form\" value to 'pizza' without changing any of the other values that you have set in the object."
        *//*
        );*/

        $ind = 0;
        $process_data = array();
        $count = 0;
        $instructions = implode(' ', $instructions);
        $instruction_size = self::count_tokens($instructions, self::$model);
        $limit = self::get_api_rate_limits(self::$model, $doing_ajax); // TODO: Setup batch processing!
        $already_processed = self::get_processed_anchor_sentences(self::$origin_post, true); //Currently only tracking sentences... not the sentence + targets that they might point to

        $queued = get_transient('wpil_queued_up_ai_sentences');
        if(!empty($queued)){
            foreach($queued as $pid => $sentences){
                $bits = explode('_', $pid);
                $queue_post = new Wpil_Model_Post($bits[1], $bits[0]);
                foreach($sentences as $sentence => $bool){
                    self::log_processed_anchor_sentence($sentence, $queue_post);
                    unset($queued[$pid][$sentence]);
                    if(empty($queued[$pid])){
                        unset($queued[$pid]);
                    }
                }
            }
        }else{
            $queued = array();
        }

        foreach($phrase_data as $sentence => $data){ // sentence == text && target_data == array(pid, post title, keywords) 
            // normalize the sentence to make sure that we're consistent
            //$sentence = self::normalize_whitespace($sentence);

            foreach($data as $target_data){
                // exit if we've been at this for more than 20 seconds or we've managed to hit the limit of posts we can process
                if(microtime(true) - $time > 20 || $count >= self::$batch_limit || $doing_ajax && $count >= self::$concurrency){
                    break;
                }

                $sentence_id = md5($sentence);
                $sentence_post_id = md5($sentence . '|||' . $target_data[0]); // we need to be able to identify the sentence and the post that it's pointing to, so use the custom id
                //$id = ($sentence . '|||' . $target_data[0]);
                $id = md5($sentence . '|||' . $target_data[0]);
                //$id = md5($sentence);

                if(isset($already_processed[$sentence_id])/* || isset($queued[$id])*/){
                    continue;
                }
                // add the sentence id to the queue list
                if(!isset($queued[$target_data[0]])){
                    $queued[$origin_post_id] = array();
                }
                $queued[$origin_post_id][$sentence_id] = true;

                if( isset($process_data[$ind]) && 
                    !empty($process_data[$ind]) &&
                    ($instruction_size + array_reduce($process_data[$ind], 
                        function($count, $chunk) use ($instruction_size){ 
                            return $count += (self::count_tokens(wp_slash((string) wp_json_encode($chunk)), self::$model)); 
                    }) > 7800 || count($process_data[$ind]) > 1)
                )
                {
                    $ind++;
                }

                // make sure there's no bad stuff in the title or keywords
                $post_title = self::trim_text_to_token_limit(mb_ereg_replace("\n", '', mb_ereg_replace('(([a-zA-Z\-_0-9]+="[^"]*")+?[\s]?)', '', strip_tags($target_data[1], '<h1><h2><h3><h4><h5><h6><title><ul><ol><li>'))), self::$model, 7800);
                $keywords = self::trim_text_to_token_limit(mb_ereg_replace("\n", '', mb_ereg_replace('(([a-zA-Z\-_0-9]+="[^"]*")+?[\s]?)', '', strip_tags($target_data[2], '<h1><h2><h3><h4><h5><h6><title><ul><ol><li>'))), self::$model, 7800);

                $content = ['meta_id' => $sentence_post_id, 'words' => '', 'sentence' => $sentence, 'post_title' => $post_title, 'keywords' => $keywords, 'match_score' => 0, 'free_form' => '', 'rando' => time()];
                $question = wp_slash((string) wp_json_encode($content));
                $query = $instruction_size + self::count_tokens($question, self::$model);
                if($token_size + $query < $limit){
                    $token_size += $query;
                    $process_data[$ind][] = $content;
                    self::$anchor_assessment_ids[$sentence_post_id] = array(
                        'sentence_id' => $sentence_id,
                        'sentence' => $sentence,
                        'pid' => $target_data[0]
                    );
                }else{
                    break;
                }

                $count++;
            }
        }

        if(!empty($process_data)){
            foreach($process_data as $key => $dat){
                $process_data[$key] = wp_slash((string) wp_json_encode($dat));
            }
        }

        // if we have sentences to process
        if(!empty($queued)){
            // save them here in case there's an error
            set_transient('wpil_queued_up_ai_sentences', $queued, DAY_IN_SECONDS * 7);
        }

        if(self::$ai_service_connected){
            self::live_query_linkwhisper_ai_results($process_data, 'completions');
            return;
        }

        if($doing_ajax){
            self::live_query_ai_results($process_data, $instructions, 'completions');
        }else{
            self::start_batch_process($process_data, $instructions, 'completions');
        }

        // if we have no process data... We must be finished!
        return empty($process_data) ? true: false;
    }

    public static function get_all_batch_process_posts($database = '', $process = 0){
        $posts = array();
        $count = 0;

        if(empty($database) || empty($process)){
            return $posts;
        }

        // get all the posts that have already been processed
        $inserted = self::get_inserted_post_data($database);

        // get all of the post data for known batches which are being processed
        $batched = (defined('DOING_AJAX') && DOING_AJAX) ? array(): self::get_batch_log_data(false, $process);

        // get all of the post and term ids that we're set to process
        $post_ids = Wpil_Report::get_all_post_ids('ai');
        $term_ids = Wpil_Report::get_all_term_ids();

        // if there are posts currently in a batch process
        if(!empty($batched)){
            // go over each process
            foreach($batched as $batch){
                // decompress the specific post data
                $dat = Wpil_Toolbox::json_decompress($batch->batch_data);
                // if that worked
                if(!empty($dat)){
                    // go over each post in the record
                    foreach($dat as $d){
                        // and if it's not in the 'completed' list
                        if(!isset($inserted[$d])){
                            // add it
                            $inserted[$d] = true;
                        }
                    }
                }
            }
        }

        if(!empty($post_ids)){
            foreach($post_ids as $post_id){
                if($count >= self::$batch_limit){ // todo: think about if we should not be limiting the batch size here. It should be no negative, but something to think about
                    break;
                }

                $id = 'post_' . $post_id;
                if(!isset($inserted[$id])){
                    $posts[] = new Wpil_Model_Post($post_id);
                    $count++;
                }
            }
        }

        if(!empty($term_ids) && $count < self::$batch_limit){
            foreach($term_ids as $term_id){
                if($count >= self::$batch_limit){
                    break;
                }

                $id = 'term_' . $term_id;
                if(!isset($inserted[$id])){
                    $posts[] = (new Wpil_Model_Post($term_id, 'term'));
                    $count++;
                }
            }
        }
        
        return $posts;
    }

    /**
     * Saves the post summary data from the batch process response. TODO: add product saving!
     * @return int Returns the total number of processed and inserted posts
     **/
    public static function save_site_post_summaries($data = array()){
        global $wpdb;
        $summary_table = $wpdb->prefix . 'wpil_ai_post_data';

        if(empty($data)){
            return 0;
        }

        if(!is_array($data)){
            $data = array($data);
        }

        $total_count = 0;
        $count = 0;
        $inserted = self::get_inserted_post_data('post-summarizing');
        $insert_query = "INSERT INTO {$summary_table} (post_id, post_type, data_type, summary, process_time, model_version) VALUES ";
        $summary_data = array();
        $place_holders = array();
        $total = count($data);
        $limit = 1000;
        foreach($data as $key => $dat){
            $total_count++;
            $dat = self::decode($dat);

            if( empty($dat) ||                              // if there's no data
                !isset(                                     // or we don't have all the data from OAI that we need
                    $dat->response,
                    $dat->custom_id,
                    $dat->response->status_code,
                    $dat->response->body,
                    $dat->response->body->choices,
                    $dat->response->body->choices[0],
                    $dat->response->body->choices[0]->message,
                    $dat->response->body->choices[0]->message->content,
                    $dat->response->body->model) || 
                (int)$dat->response->status_code !== 200 || // or this wasn't a success
                isset($inserted[$dat->custom_id])           // or we've already saved the post
            ){
                continue;
            }

            $results = self::decode($dat->response->body->choices[0]->message->content); // the response is supposed to be in JSON, keyed to the "keywords" index

            if(!empty($results) && isset($results->results) && !empty($results->results)){
                $results = $results->results;
            }

            if(empty($results) || !is_array($results) || !isset($results[0]->summary) || empty($results[0]->summary)){
                if(!empty($results) && isset($results->summary) && !empty($results->summary)){
                    if(is_string($results->summary)){
                        $summary = trim(sanitize_text_field(trim($results->summary)));
                    }elseif(is_object($results->summary) || is_array($results->summary)){
                        $summary = sanitize_text_field(json_encode($results));
                    }else{
                        continue;
                    }
                }elseif(!empty($results) && (is_object($results) || is_array($results))){
                    $summary = json_encode(array_map('sanitize_text_field', (array)$results)); // if the response is an object, format it so we can catch it in the results
                }else{
                    continue;
                }
            }else{
                $summary = trim(sanitize_text_field(trim($results[0]->summary)));
            }

            $ids = explode('_', $dat->custom_id);
            array_push(
                $summary_data, 
                $ids[1],
                $ids[0],
                (($ids[0] === 'post') ? 1: 0),
                $summary,
                time(),
                $dat->response->body->model
            );
            $place_holders[] = "('%d', '%s', '%d', '%s', '%d', '%s')";

            // if we've hit the limit
            if($count > $limit || ($key + 1) >= $total){
                // assemble the insert
                $insert = ($insert_query . implode(', ', $place_holders));
                $insert = $wpdb->prepare($insert, $summary_data);
                // insert the data
                $wpdb->query($insert);
                // reset the data variables
                $summary_data = [];
                $place_holders = [];
                $count = 0;
            }

            $count++;
        }

        // if we still have data that hasn't been inserted
        if(!empty($summary_data) && !empty($place_holders)){
            // assemble the insert
            $insert = ($insert_query . implode(', ', $place_holders));
            $insert = $wpdb->prepare($insert, $summary_data);
            // and insert the data
            $wpdb->query($insert);
        }

        // return the total number of posts processed
        return $total_count;
    }

    /**
     * Saves the embedding data from the batch process response
     * @return int Returns the total number of processed and inserted posts
     **/
    public static function save_site_embeddings($data = array()){
        global $wpdb;
        $embedding_table = $wpdb->prefix . 'wpil_ai_embedding_data';

        if(empty($data)){
            return 0;
        }

        if(!is_array($data)){
            $data = array($data);
        }

        $total_count = 0;
        $count = 0;
        $inserted = self::get_inserted_post_data('create-post-embeddings');
        $insert_query = "INSERT INTO {$embedding_table} (post_id, post_type, data_type, embed_data, process_time, model_version) VALUES ";
        $embedding_data = array();
        $place_holders = array();
        $total = count($data);
        $limit = 1000;
        foreach($data as $key => $dat){
            $total_count++;
            $dat = self::decode($dat);

            if( empty($dat) ||                              // if there's no data
                !isset(                                     // or we don't have all the data from OAI that we need
                    $dat->response,
                    $dat->custom_id,
                    $dat->response->status_code,
                    $dat->response->body,
                    $dat->response->body->data,
                    $dat->response->body->data[0],
                    $dat->response->body->data[0]->embedding,
                    $dat->response->body->model) || 
                (int)$dat->response->status_code !== 200 || // or this wasn't a success
                isset($inserted[$dat->custom_id])           // or we've already saved the post
            ){
                continue;
            }

            $ids = explode('_', $dat->custom_id);
            array_push(
                $embedding_data, 
                $ids[1],
                $ids[0],
                (($ids[0] === 'post') ? 1: 0),
                Wpil_Toolbox::json_compress($dat->response->body->data[0]->embedding, true),
                time(),
                $dat->response->body->model
            );
            $place_holders[] = "('%d', '%s', '%d', '%s', '%d', '%s')";

            // if we've hit the limit
            if($count > $limit || ($key + 1) >= $total){
                // assemble the insert
                $insert = ($insert_query . implode(', ', $place_holders));
                $insert = $wpdb->prepare($insert, $embedding_data);
                // insert the data
                $wpdb->query($insert);
                // reset the data variables
                $embedding_data = [];
                $place_holders = [];
                $count = 0;
            }

            $count++;
        }

        // if we still have data that hasn't been inserted
        if(!empty($embedding_data) && !empty($place_holders)){
            // assemble the insert
            $insert = ($insert_query . implode(', ', $place_holders));
            $insert = $wpdb->prepare($insert, $embedding_data);
            // and insert the data
            $wpdb->query($insert);
        }

        // return the total number of posts processed
        return $total_count;
    }

    /**
     * Saves an empty dataset for a post with no content so we can process it and check it off the list
     * @return int Returns the total number of processed and inserted posts
     **/
    public static function save_empty_post_embedding($id = '', $model = ''){
        global $wpdb;
        $embedding_table = $wpdb->prefix . 'wpil_ai_embedding_data';

        if(empty($id)){
            return 0;
        }

        $bits = explode('_', $id);

        if(empty($bits)){
            return 0;
        }

        $post_id = $bits[1];
        $post_type = $bits[0];

        // check to make sure that the post isn't already saved
        if(!empty($wpdb->get_var($wpdb->prepare("SELECT COUNT(*) FROM $embedding_table WHERE `post_id` = %d AND `post_type` = %s LIMIT 1", $post_id, $post_type)))){
            return 1;
        }

        // create the "empty" embedding calculation
        $data = [-0.0011765664, -0.034535293, -0.057125367, 0.02252959, -0.07118746, -0.014417427, 0.018023673, 0.14273782, 0.0019250326, 0.027806656, -0.051893663, 0.050804984, -0.05035137, 0.026551653, -0.0033775487, -0.003103489, -0.054796804, 0.0131699825, -0.056218136, 0.03662193, 0.035442524, -0.02615852, -0.12410932, 0.061268393, 0.02331586, -0.07366723, -0.024298694, 0.011975461, -0.03163215, 0.0028124189, 0.05062354, 0.056278616, -0.003301946, 0.052226316, -0.030346906, 0.011605008, -0.01905187, -0.06610696, -0.010047593, 0.026869183, -0.02131995, -0.0042224084, -0.036198553, 0.038527112, -0.0027689473, 0.043758817, 0.035714693, -0.0022794202, 0.013555557, 0.015574147, -0.037408195, -0.06096598, -0.01537002, -0.0027122453, -0.03758964, -0.04067423, 0.025326889, -0.09997695, -0.006762658, -0.04354713, 0.016965237, -0.0013901439, -0.023996282, 0.021274587, 0.019248437, 0.0005570971, 0.0046306625, -0.031511188, 0.033779267, 0.09695285, 0.017207164, 0.01994398, 0.019581089, 0.06574407, 0.0153549, -0.040039167, 0.031571668, 0.021501396, -0.012625644, 0.014939085, 0.005435831, -0.06919155, -0.029711844, 0.053829093, -0.022937845, 0.004388734, -0.00711043, -0.23987211, 0.025508337, 0.022816882, 0.026899425, 0.010607053, 0.061842974, -0.022665676, 0.03867832, -0.037196506, -0.03870856, 0.005352668, -0.08370726, -0.0040334016, 0.061540563, -0.02987817, 0.050925948, 0.011068229, -0.02871389, 0.040281095, -0.026506292, 0.035200596, -0.01930892, 0.0074657626, -0.013956251, -0.019565968, -0.035442524, -0.0092084035, -0.053556923, 0.008013882, -0.03589614, -0.053617403, -0.0312995, -0.021864288, 0.042821344, -0.07263903, -0.013797485, 0.03099709, -0.005946149, -0.014341824, -0.06634889, -0.03214625, -0.08346533, 0.03263011, -0.031450704, -0.011234554, -0.04750871, 0.04581521, -0.019550847, -0.045210388, -0.038950488, 0.063143335, -0.084493525, -0.053375475, -0.06117767, 0.015997522, 0.011899858, 0.002109314, -0.030815642, -0.039071452, 0.038950488, -0.006486708, 0.010304642, -0.03786181, 0.057125367, -0.010387805, 0.016299933, -0.016194088, 0.036954578, 0.016904755, 0.020563923, -0.059635375, 0.01820512, 0.057911634, -0.014470348, 0.006921423, -0.0683448, 0.030195702, -0.036863856, -0.02847196, 0.021486275, -0.014886163, 0.023225136, 0.03934362, -0.043153998, 0.0683448, 0.047206298, -0.008724547, 0.009782984, 0.097860076, -0.0051182997, 0.014765199, 0.05942369, -0.0066719344, 0.09477549, 0.026385328, 0.025961952, 0.007087749, -0.0106524145, -0.015135651, -0.009760303, 0.060119234, -0.034565534, -0.09864634, -0.0007446862, -0.07868724, -0.0075224643, -0.09017885, 0.06265948, -0.021985253, -0.013427032, 0.008550661, -0.018053913, 0.013910889, 0.024601104, 0.028230032, 0.028124189, -0.11691195, 0.046480514, -0.043940265, -0.019233316, 0.015725352, -0.0018758909, 0.05863742, -0.024434779, -0.08818294, -0.031148294, -0.024706949, -0.0072124936, 0.1143717, -0.07421157, -0.011771333, 0.016345294, 0.007991201, -0.0010820631, -0.009442772, 0.049686067, 0.06695371, -0.051561013, -0.044030987, 0.02479767, 0.06519973, 0.026113158, 0.016738428, -0.07729615, -0.02934895, -0.031692635, 0.07094553, -0.04896028, -0.0063657435, 0.040825434, -0.018386565, 0.071550354, 0.06265948, -0.020337114, 0.012398835, -0.030150339, -0.05032113, 0.049686067, -0.040613748, -0.047750637, -0.046843406, -0.0069592246, -0.031239018, 0.02984793, 0.012550041, 0.084493525, 0.014750078, 0.04155122, 0.022741279, -0.033718783, 0.025190806, -0.015294418, -0.013411911, 0.05975634, 0.06483684, 0.028305635, 0.03867832, 0.01849241, 0.012119106, -0.048718352, -0.07971544, -0.006637913, -0.051349323, 0.06265948, -0.0046835844, -0.016753549, -0.017328128, 0.042851586, 0.024873273, -0.019959101, 0.020291753, 0.010629733, -0.0027443764, -0.007741712, 0.009246205, 0.0690101, -0.026506292, -0.03641024, -0.0051069595, -0.081892796, -0.01735837, -0.03961579, -0.0035155236, -0.041460495, -0.016133606, 0.04327496, 0.06380864, -0.021561878, 0.036228795, 0.020367356, 0.00072484044, -0.050169922, 0.008512859, 0.08364678, -0.0575185, 0.01880994, -0.060179714, 0.07094553, 0.04642003, 0.09804153, 0.020881454, -0.006868501, 0.07269952, 0.047659915, -0.06574407, -0.016103366, -0.021970132, -0.054252468, -0.0318136, -0.0016056114, -0.02761009, 0.022317905, -0.029802566, 0.03834567, -0.040281095, -0.0984649, 0.023784596, 0.08134846, 0.022378387, -0.020049825, -0.021153623, 0.00938229, 0.0092235245, -0.057669707, -0.013805045, 0.008384335, 0.04267014, 0.045452315, -0.10257769, -0.012013262, 0.011476483, -0.01026684, -0.045694247, -0.066167444, 0.009835905, 0.0380735, -0.030543473, -0.042035077, -0.035714693, 0.079533994, -0.021607239, -0.016315054, -0.0015895459, 0.0020904134, -0.07439301, -0.029091902, -0.031087812, 0.0910256, -0.026854064, 0.012035943, -0.01451571, 0.035170358, -0.013706761, -0.015430503, 0.00003139282, 0.07935255, 0.062115144, -0.051923905, -0.08364678, -0.04155122, 0.019747414, -0.01310194, 0.022378387, -0.120298944, 0.05897007, -0.00032934407, -0.00883795, -0.08388871, -0.04705509, 0.06005875, -0.020594163, -0.0043282523, -0.039676275, -0.047357503, 0.0426399, 0.017963191, 0.026037555, 0.05996803, -0.12096425, -0.0016235671, -0.03870856, 0.029273348, -0.003046787, -0.059363205, -0.0074960035, -0.020246392, 0.025523458, 0.07312289, -0.019702053, -0.0609055, -0.008656505, -0.04635955, 0.09628754, 0.016481379, -0.07439301, -0.019278677, -0.009147922, 0.011536965, -0.05996803, -0.012988537, 0.019883499, 0.035230838, 0.010720457, -0.053587165, -0.011597447, 0.03816422, -0.051288843, 0.043577373, 0.027141353, 0.013449713, 0.017963191, 0.07463494, -0.04155122, 0.05721609, -0.025478095, -0.0112043135, 0.0038670758, -0.04723654, 0.11310157, 0.038043257, -0.028683648, -0.03529132, -0.021501396, -0.07663085, 0.0210629, 0.0056437384, -0.008611143, -0.03786181, -0.04233749, 0.012345914, -0.061026465, -0.0058818865, 0.022711039, -0.008467497, -0.0072654155, -0.011234554, 0.0312995, -0.014379625, 0.05325451, -0.008633823, -0.048294976, 0.0013334418, 0.025175685, 0.002956064, -0.036833614, -0.0443334, -0.026687738, 0.0031809818, 0.033204686, 0.059937786, 0.16185017, 0.009132801, -0.009132801, 0.034837704, 0.024963997, 0.06278045, 0.0066530337, 0.037438437, -0.00015144158, -0.039373863, -0.00910256, -0.034232885, -0.008701866, 0.08364678, -0.025478095, -0.023013448, -0.0040334016, -0.032116007, -0.042216524, 0.036561444, -0.01735837, -0.0015904909, -0.08044123, -0.014349384, -0.03018058, -0.03016546, -0.023270497, 0.05271017, 0.008679185, 0.038466632, 0.018885544, -0.023361221, -0.0318136, 0.062115144, -0.037740845, -0.012119106, 0.045028944, 0.029258229, 0.0020563921, -0.011801574, -0.00020554473, 0.016058004, 0.051319085, 0.006006631, 0.022060854, 0.09241669, -0.030906366, -0.036500964, -0.07911062, 0.047085334, 0.016859392, -0.037105784, 0.07463494, 0.04699461, -0.029651362];

        $wpdb->insert($embedding_table, [
            'post_id' => $post_id,
            'post_type' => $post_type,
            'data_type' => (($post_type === 'post') ? 1: 0),
            'embed_data' => Wpil_Toolbox::json_compress($data, true),
            'is_empty' => 1,
            'process_time' => time(),
            'model_version' => $model
        ]);

        // return the total number of posts processed
        return !empty($wpdb->insert_id) ? 1: 0;
    }

    /**
     * Checks to make sure that we've created embeddings for all the available posts
     **/
    public static function has_completed_post_embeddings(){
        global $wpdb;
        $table = $wpdb->prefix . 'wpil_ai_embedding_data';

        $ids = array();
        $data = $wpdb->get_results("SELECT `post_id`, `post_type` FROM {$table}");
        $not_processed = array();

        if(!empty($data)){
            foreach($data as $dat){
                if(empty($dat) || !isset($dat->post_id, $dat->post_type) || empty($dat->post_id) || empty($dat->post_type)){
                    continue;
                }
                $id = $dat->post_type . '_' . $dat->post_id;
                $ids[$id] = true;
            }
        }

        if(!empty($ids)){
            $post_ids = Wpil_Report::get_all_post_ids('ai');
            $term_ids = Wpil_Report::get_all_term_ids();

            if(!empty($post_ids)){
                foreach($post_ids as $p_id){
                    $id = 'post_' . $p_id;

                    if(!isset($ids[$id])){
                        $not_processed[] = $id;
                    }

                }
            }

            if(!empty($term_ids)){
                foreach($term_ids as $t_id){
                    $id = 'term_' . $t_id;

                    if(!isset($ids[$id])){
                        $not_processed[] = $id;
                    }

                }
            }
        }

        return empty($not_processed); // return true on empty so we know that there are no more posts to process
    }

    /**
     * Checks to make sure that we've created embedding calculations for all the available posts.
     * Only checks to see if all the embedding data has been used for calculations.
     **/
    public static function has_completed_post_embedding_calculations(){
        global $wpdb;
        $embedding_table    = $wpdb->prefix . 'wpil_ai_embedding_data';
        $calculation_table  = $wpdb->prefix . 'wpil_ai_embedding_calculation_data';

        $ids = array();
        $embedding_data     = $wpdb->get_results("SELECT `post_id`, `post_type` FROM {$embedding_table}");
        $calculation_data   = $wpdb->get_results("SELECT `post_id`, `post_type` FROM {$calculation_table}");
        $last_embedding_index = self::get_last_embedding_index();

        if(!empty($embedding_data)){
            foreach($embedding_data as $dat){
                if(empty($dat) || !isset($dat->post_id, $dat->post_type) || empty($dat->post_id) || empty($dat->post_type)){
                    continue;
                }
                $id = $dat->post_type . '_' . $dat->post_id;
                $ids[$id] = true;
            }
        }

        if(!empty($calculation_data)){
            foreach($calculation_data as $dat){
                if( empty($dat) || 
                    !isset($dat->post_id, $dat->post_type) || 
                    empty($dat->post_id) || 
                    empty($dat->post_type) || 
                    (isset($dat->calc_index) && $dat->calc_index < $last_embedding_index))
                {
                    continue;
                }
                $id = $dat->post_type . '_' . $dat->post_id;
                if(isset($ids[$id])){
                    unset($ids[$id]);
                }
            }
        }

        return empty($ids); // return true on empty so we know that there are no more posts to process
    }

    /**
     * Saves the products found during our search of the site posts
     * @return int Returns the total number of processed and inserted posts
     **/
    public static function save_site_products($data = array()){
        global $wpdb;
        $product_table = $wpdb->prefix . 'wpil_ai_product_data';

        if(empty($data)){
            return 0;
        }

        if(!is_array($data)){
            $data = array($data);
        }

        $total_count = 0;
        $count = 0;
        $inserted = self::get_inserted_post_data('product-detecting');
        $insert_query = "INSERT INTO {$product_table} (post_id, post_type, data_type, products, product_count, process_time, model_version) VALUES ";
        $product_data = array();
        $place_holders = array();
        $total = count($data);
        $limit = 1000;
        foreach($data as $key => $dat){
            $total_count++;
            $dat = self::decode($dat);

            if( empty($dat) ||                              // if there's no data
                !isset(                                     // or we don't have all the data from OAI that we need
                    $dat->response,
                    $dat->custom_id,
                    $dat->response->status_code,
                    $dat->response->body,
                    $dat->response->body->choices,
                    $dat->response->body->choices[0],
                    $dat->response->body->choices[0]->message,
                    $dat->response->body->choices[0]->message->content,
                    $dat->response->body->model) || 
                (int)$dat->response->status_code !== 200 || // or this wasn't a success
                isset($inserted[$dat->custom_id])           // or we've already saved the post
            ){
                continue;
            }

            $results = self::decode($dat->response->body->choices[0]->message->content); // the response is supposed to be in JSON, keyed to the "keywords" index

            if(!empty($results) && isset($results->results) && !empty($results->results)){
                $results = $results->results;
            }

            if(empty($results) || !is_array($results) || !isset($results[0]->products) || empty($results[0]->products)){
                if(!empty($results) && isset($results->products) && !empty($results->products)){
                    if(is_string($results->products)){
                        $products = array_map('sanitize_text_field', array_unique(explode(',', $results->products)));
                    }elseif(is_array($results->products) || is_object($results->products)){
                        $products = array_map('sanitize_text_field', array_unique((array)$results->products));
                    }else{
                        continue;
                    }
                    
                }else{
                    continue;
                }
            }else{
                $products = array_map('sanitize_text_field', array_unique(explode(',', $results[0]->products)));
            }

            $no_products = array_search('no-products', $products);
            if(false !== $no_products){
                unset($products[$no_products]);
            }

            $ids = explode('_', $dat->custom_id);
            array_push(
                $product_data, 
                $ids[1],
                $ids[0],
                (($ids[0] === 'post') ? 1: 0),
                Wpil_Toolbox::json_compress($products),
                count($products),
                time(),
                $dat->response->body->model
            );
            $place_holders[] = "('%d', '%s', '%d', '%s', '%d', '%d', '%s')";

            // if we've hit the limit
            if($count > $limit || ($key + 1) >= $total){
                // assemble the insert
                $insert = ($insert_query . implode(', ', $place_holders));
                $insert = $wpdb->prepare($insert, $product_data);
                // insert the data
                $wpdb->query($insert);
                // reset the data variables
                $product_data = [];
                $place_holders = [];
                $count = 0;
            }

            $count++;
        }

        // if we still have data that hasn't been inserted
        if(!empty($product_data) && !empty($place_holders)){
            // assemble the insert
            $insert = ($insert_query . implode(', ', $place_holders));
            $insert = $wpdb->prepare($insert, $product_data);
            // and insert the data
            $wpdb->query($insert);
        }

        // return the total number of posts processed
        return $total_count;
    }

    /**
     * Saves the products found during our search of the site posts
     * @return int Returns the total number of processed and inserted posts
     **/
    public static function save_site_keywords($data = array()){
        global $wpdb;
        $keyword_table = $wpdb->prefix . 'wpil_ai_keyword_data';
        $max_keywords = Wpil_Settings::get_ai_keyword_count_max();

        if(empty($data)){
            return 0;
        }
        
        if(!is_array($data)){
            $data = array($data);
        }

        $total_count = 0;
        $count = 0;
        $inserted = self::get_inserted_post_data('keyword-detecting');
        $insert_query = "INSERT INTO {$keyword_table} (post_id, post_type, data_type, keywords, keyword_count, process_time, model_version) VALUES ";
        $keyword_data = array();
        $place_holders = array();
        $total = count($data);
        $limit = 1000;
        foreach($data as $key => $dat){
            $total_count++;
            $dat = self::decode($dat);

            if( empty($dat) ||                              // if there's no data
                !isset(                                     // or we don't have all the data from OAI that we need
                    $dat->response,
                    $dat->custom_id,
                    $dat->response->status_code,
                    $dat->response->body,
                    $dat->response->body->choices,
                    $dat->response->body->choices[0],
                    $dat->response->body->choices[0]->message,
                    $dat->response->body->choices[0]->message->content,
                    $dat->response->body->model) || 
                (int)$dat->response->status_code !== 200 || // or this wasn't a success
                isset($inserted[$dat->custom_id])           // or we've already saved the post
            ){
                continue;
            }

            $results = self::decode($dat->response->body->choices[0]->message->content); // the response is supposed to be in JSON, keyed to the "keywords" index

            if(!empty($results) && isset($results->results) && !empty($results->results)){
                $results = $results->results;
            }
            if(empty($results) || !is_array($results) || !isset($results[0]->keywords) || empty($results[0]->keywords)){
                if(!empty($results) && isset($results->keywords) && !empty($results->keywords)){
                    if(is_array($results->keywords) || is_object($results->keywords)){
                        $kwrds = (array) $results->keywords;
                    }else{
                        $kwrds = explode(',', $results->keywords);
                    }
                    $keywords = array_map('sanitize_text_field', array_unique($kwrds));
                }else{
                    continue;
                }
            }else{
                $keywords = array_map('sanitize_text_field', array_unique(explode(',', $results[0]->keywords)));
            }

            $no_keywords = array_search('no-keywords', $keywords);
            if(false !== $no_keywords){
                unset($keywords[$no_keywords]);
            }

            // if there are more keywords available than the user's limit
            if(count($keywords) > $max_keywords){
                // make sure all of the keywords are unique and trim to fit
                $keywords = array_slice(array_unique($keywords), 0, $max_keywords);
            }

            $ids = explode('_', $dat->custom_id);
            array_push(
                $keyword_data, 
                $ids[1],
                $ids[0],
                (($ids[0] === 'post') ? 1: 0),
                Wpil_Toolbox::json_compress($keywords),
                count($keywords),
                time(),
                $dat->response->body->model
            );
            $place_holders[] = "('%d', '%s', '%d', '%s', '%d', '%d', '%s')";

            // if we've hit the limit
            if($count > $limit || ($key + 1) >= $total){
                // assemble the insert
                $insert = ($insert_query . implode(', ', $place_holders));
                $insert = $wpdb->prepare($insert, $keyword_data);
                // insert the data
                $wpdb->query($insert);
                // reset the data variables
                $keyword_data = [];
                $place_holders = [];
                $count = 0;
            }

            $count++;
        }

        // if we still have data that hasn't been inserted
        if(!empty($keyword_data) && !empty($place_holders)){
            // assemble the insert
            $insert = ($insert_query . implode(', ', $place_holders));
            $insert = $wpdb->prepare($insert, $keyword_data);
            // and insert the data
            $wpdb->query($insert);
        }

        // return the total number of posts processed
        return $total_count;
    }

    /**
     * Saves empty data in the AI tables so we can process posts without content
     **/
    public static function save_empty_post_data($id = '', $model = '', $purpose = ''){
        global $wpdb;
        $summary_table = $wpdb->prefix . "wpil_ai_post_data";
        $product_table = $wpdb->prefix . "wpil_ai_product_data";
        $keyword_table = $wpdb->prefix . "wpil_ai_keyword_data";
        
        if(empty($id)){
            return 0;
        }

        $bits = explode('_', $id);

        if(empty($bits)){
            return 0;
        }

        $post_id = $bits[1];
        $post_type = $bits[0];

        if(empty($purpose) || $purpose === 'create-post-embeddings'){
            self::save_empty_post_embedding($id, $model);
        }
        // TODO: Uncomment if we ever implement post summaries
/*
        if(empty($purpose) || $purpose === 'post-summarizing' || false !== strpos($purpose, 'summary')){
            if(empty($wpdb->get_var($wpdb->prepare("SELECT COUNT(*) FROM {$summary_table} WHERE `post_id` = %d AND `post_type` = %s LIMIT 1", $post_id, $post_type)))){
                $wpdb->insert($summary_table, [
                    'post_id' => $post_id,
                    'post_type' => $post_type,
                    'data_type' => (($post_type === 'post') ? 1: 0),
                    'summary' => '',
                    'process_time' => time(),
                    'model_version' => $model
                ]);
            }
        }*/

        if(empty($purpose) || false !== strpos($purpose, 'product')){
            if(empty($wpdb->get_var($wpdb->prepare("SELECT COUNT(*) FROM {$product_table} WHERE `post_id` = %d AND `post_type` = %s LIMIT 1", $post_id, $post_type)))){
                $wpdb->insert($product_table, [
                    'post_id' => $post_id,
                    'post_type' => $post_type,
                    'data_type' => (($post_type === 'post') ? 1: 0),
                    'products' => Wpil_Toolbox::json_compress(array()),
                    'product_count' => 0,
                    'process_time' => time(),
                    'model_version' => $model
                ]);
            }
        }

        if(empty($purpose) || false !== strpos($purpose, 'keyword')){
            if(empty($wpdb->get_var($wpdb->prepare("SELECT COUNT(*) FROM {$keyword_table} WHERE `post_id` = %d AND `post_type` = %s LIMIT 1", $post_id, $post_type)))){
                $wpdb->insert($keyword_table, [
                    'post_id' => $post_id,
                    'post_type' => $post_type,
                    'data_type' => (($post_type === 'post') ? 1: 0),
                    'keywords' => Wpil_Toolbox::json_compress(array()),
                    'keyword_count' => 0,
                    'process_time' => time(),
                    'model_version' => $model
                ]);
            }
        }

        // return the total number of posts processed
        return !empty($wpdb->insert_id) ? 1: 0;
    }

    /**
     * Saves the embedding data for the sentences for a single post.
     * @param Wpil_Model_Post $post
     * @param array $data The embedding data for the post. 
     **/
    public static function save_single_post_embedding_data($post, $data = array()){
        global $wpdb;
        $embedding_table = $wpdb->prefix . 'wpil_ai_embedding_phrase_data';

        if(empty($post) || !is_a($post, 'Wpil_Model_Post') || empty($data)){
            return 0;
        }

        if(!is_array($data)){
            $data = array($data);
        }

        $model = Wpil_Settings::getChatGPTVersion('create-post-embeddings');
        $dimension_count =  0;
        $compressed_data = array();
        foreach($data as $phrase => $dat){
            $compressed_data[$phrase] = $dat[0]->embedding;
            $dimension_count = count($dat[0]->embedding);
        }

        $insert_query = "INSERT INTO {$embedding_table} (post_id, post_type, data_type, post_phrase_id, embed_data, no_data, process_time, model_version, dimension_count) VALUES ";
        $embedding_data = array(
            $post->id,
            $post->type,
            (($post->type === 'post') ? 1: 0),
            Wpil_Toolbox::create_post_content_id($post),
            Wpil_Toolbox::json_compress($compressed_data),
            ((empty($compressed_data)) ? 1: 0),
            time(),
            $model,
            $dimension_count
        );
        $place_holders[] = "('%d', '%s', '%d', '%s', '%s', '%d', '%d', '%s', '%d')";
        
        // assemble the insert
        $insert = ($insert_query . implode(', ', $place_holders));
        $insert = $wpdb->prepare($insert, $embedding_data);
        // and insert the data
        $wpdb->query($insert);

        // return the total number of posts processed
        return (!empty($wpdb->last_error)) ? 1 : 0;
    }

    /**
     * Saves an empty dataset for a post with no content so we can process it and check it off the list
     * @return int Returns the total number of processed and inserted posts
     **/
    public static function save_empty_phrase_embedding($id = '', $model = ''){
        global $wpdb;
        $embedding_table = $wpdb->prefix . 'wpil_ai_embedding_phrase_data';

        if(empty($id)){
            return 0;
        }

        $bits = explode('_', $id);

        if(empty($bits)){
            return 0;
        }

        $post_id = $bits[1];
        $post_type = $bits[0];

        // check to make sure that the post isn't already saved
        if(!empty($wpdb->get_var($wpdb->prepare("SELECT COUNT(*) FROM $embedding_table WHERE `post_id` = %d AND `post_type` = %s LIMIT 1", $post_id, $post_type)))){
            return 1;
        }

        // create the "empty" embedding calculation
        $data = array();

        $wpdb->insert($embedding_table, [
            'post_id' => $post_id,
            'post_type' => $post_type,
            'data_type' => (($post_type === 'post') ? 1: 0),
            'embed_data' => Wpil_Toolbox::json_compress($data),
            'no_data' => 1,
            'process_time' => time(),
            'model_version' => $model
        ]);

        // return the total number of posts processed
        return !empty($wpdb->insert_id) ? 1: 0;
    }

    /**
     * Saves AI generated suggestions to their own table.
     * @return int Returns the total number of processed and inserted suggestions
     **/
    public static function save_ai_suggestion_words($data = array()){
        global $wpdb;
        $anchor_table = $wpdb->prefix . 'wpil_ai_suggested_anchors';
        $enable_notes = false;

        if(empty($data) || empty(self::$origin_post)){
            return 0;
        }
        
        if(!is_array($data)){
            $data = array($data);
        }

        $total_count = 0;
        $count = 0;
        $insert_query = "INSERT INTO {$anchor_table} (post_id, post_type, data_type, sentence_post_id, sentence_id, suggestion_words, notes, target_id, target_type, target_data_type, link_score, ignore_suggestion, process_time, model_version) VALUES ";
        $inserted = self::get_processed_anchor_sentences(self::$origin_post, true);
        $suggestion_data = array();
        $place_holders = array();
        $limit = 1000;
        foreach($data as $key => $dat){
            $total_count++;
            $dat = self::decode($dat);

            if( empty($dat) ||                              // if there's no data
                !isset(                                     // or we don't have all the data from OAI that we need
                    $dat->response,
                    $dat->custom_id,
                    $dat->response->status_code,
                    $dat->response->body,
                    $dat->response->body->choices,
                    $dat->response->body->choices[0],
                    $dat->response->body->choices[0]->message,
                    $dat->response->body->choices[0]->message->content,
                    $dat->response->body->model) || 
                (int)$dat->response->status_code !== 200 // or this wasn't a success
            ){
                continue;
            }

            $results = self::decode($dat->response->body->choices[0]->message->content); // the response is supposed to be in JSON, keyed to the "keywords" index

            if(!empty($results) && isset($results->results) && !empty($results->results)){
                $results = $results->results;
            }

            if(empty($results)){
                continue;
            }

            // get the processing transient
            $queued = get_transient('wpil_queued_up_ai_sentences');
            $total = count($data);
            foreach($results as $res){
                // if we can get the sentence id information from our tracker
                if(isset(self::$anchor_assessment_ids[$res->meta_id])){
                    $sentence_post_id = $res->meta_id; // we need to be able to identify the sentence and the post that it's pointing to, so use the custom id
                    $sentence_id = self::$anchor_assessment_ids[$res->meta_id]['sentence_id']; // we also need to be able to id the sentence within the post quickly
                    $sentence = self::$anchor_assessment_ids[$res->meta_id]['sentence']; // get the real sentencce!
                    $pid = self::$anchor_assessment_ids[$res->meta_id]['pid'];
                }elseif(false !== strpos($res->meta_id, '_')){
                    // if we can't try our best with the return data
                    $sentence_post_id = md5($res->sentence . '|||' . $res->meta_id); // we need to be able to identify the sentence and the post that it's pointing to, so use the custom id
                    $sentence_id = md5($res->sentence); // we also need to be able to id the sentence within the post quickly
                    $sentence = $res->sentence;
                    $pid = $res->meta_id;
                }else{
                    // come up with a way of marking the post completed, because there's no way to tell if we're done or not...
                    $sentence_post_id = null;
                }

                //$id = ($sentence . '|||' . $pid);


                // skip this sentence if we've already processed it
                if(isset($inserted[$sentence_id])){
                    continue;
                }

                // log the sentence's id
                self::log_processed_anchor_sentence($sentence, self::$origin_post);

                // unset the sentence from the processing transient
                if(isset($queued[$pid], $queued[$pid][$sentence_id])){
                    unset($queued[$pid][$sentence_id]);
                }

                $target_ids = explode('_', $pid);

                if(empty($target_ids)){
                    continue;
                }

                $target_post = new Wpil_Model_Post($target_ids[1], $target_ids[0]);

                if(empty($res->words) || !isset($res->words)){
                    $res->words = 'no-words';
                }

                foreach($res as $ind => $value){
                    if(empty($res->$ind) || !isset($res->$ind)){
                        $res->$ind = '';
                    }
                    $res->$ind = sanitize_text_field($value);
                }

                $match_score = (isset($res->match_score) && !empty($res->match_score)) ? (int)$res->match_score: 0;

                array_push(
                    $suggestion_data, 
                    self::$origin_post->id,
                    self::$origin_post->type,
                    ((self::$origin_post->type === 'post') ? 1: 0),
                    (!empty($sentence_post_id) ? $sentence_post_id: null),
                    $sentence_id,
                    Wpil_Toolbox::json_compress($res->words),
                    ($enable_notes) ? $res->free_form: '', //Wpil_Toolbox::json_compress($res->free_form): '',
                    $target_post->id,
                    $target_post->type,
                    (($target_post->type === 'post') ? 1: 0),
                    $match_score,
                    ($match_score < 5) ? 1: 0,
                    time(),
                    $dat->response->body->model
                );

                $place_holders[] = "('%d', '%s', '%d', '%s', '%s', '%s', '%s', '%d', '%s', '%d', '%d', '%d', '%d', '%s')";

                // if we've hit the limit
                if($count > $limit || ($key + 1) >= $total){
                    // assemble the insert
                    $insert = ($insert_query . implode(', ', $place_holders));
                    $insert = $wpdb->prepare($insert, $suggestion_data);
                    // insert the data
                    $wpdb->query($insert);
                    // reset the data variables
                    $suggestion_data = [];
                    $place_holders = [];
                    $count = 0;
                }

                $count++;
            }

            // if we still have data that hasn't been inserted
            if(!empty($suggestion_data) && !empty($place_holders)){
                // assemble the insert
                $insert = ($insert_query . implode(', ', $place_holders));
                $insert = $wpdb->prepare($insert, $suggestion_data);
                // and insert the data
                $wpdb->query($insert);
            }

            if(!empty($queued)){
                set_transient('wpil_queued_up_ai_sentences', $queued, DAY_IN_SECONDS * 7);
            }else{
                delete_transient('wpil_queued_up_ai_sentences');
            }
        }

        // return the total number of phrases processed
        return $total_count;
    }

    /**
     * Saves AI generated LINING suggestions to their own table.
     * TOTALLY DIFFERENT FROM THE LAST IMPLEMENTATION!!!
     * @return int Returns the total number of processed and inserted suggestions
     **/
    public static function save_ai_linking_suggestions($data = array(), $process_key = ''){
        global $wpdb;
        $linking_table = $wpdb->prefix . 'wpil_ai_linking';

        if(empty($data)){
            return 0;
        }
        
        if(!is_array($data)){
            $data = array($data);
        }

        if(empty($process_key)){
            if(!empty(self::$active_linking_process_key)){
                $process_key = self::$active_linking_process_key;
            }else{
                $process_key = get_option('wpil_ai_linking_process_key', '');
            }
        }
        $process_key = is_string($process_key) ? sanitize_text_field($process_key) : '';
        $orphan_only_targets = self::is_orphan_fix_process_key($process_key);

        $total_count = 0;
        $count = 0;
        $cols = [   'post_id', // cause I'm sick of long strings!
                    'post_type',
                    'data_type',
                    'target_id',
                    'target_type',
                    'target_data_type',
                    'sentence_text',
                    'sentence_id',
                    'sentence_with_anchor_text',
                    'ai_relation_score',
                    'ignored',
                    'process_key',
                    'process_time',
                    'model_version'
                ];
        $insert_query = "INSERT INTO {$linking_table} (" . implode(',', $cols) .") VALUES ";
        $suggestion_data = array();
        $place_holders = array();
        $total = count($data);
        $limit = 1000;
        $ai_relation_data_cache = array();
        foreach($data as $key => $dat){
            $total_count++;

            $unwrapped = self::unwrap_linking_batch_item($dat);
            if(!$unwrapped['ok'] || empty($unwrapped['custom_id'])){
                continue;
            }

            $results = $unwrapped['results'];
            $model   = $unwrapped['model'];

            if(!empty($results) && isset($results->results) && !empty($results->results)){
                $results = $results->results;
            }elseif(!empty($results) && isset($results[0]) && !empty($results[0])){
                $results = $results[0];
            }

            if(empty($results)){
                continue;
            }

            $source_post_bits = explode('_', $unwrapped['custom_id']);
            if(!isset($source_post_bits[0], $source_post_bits[1])){
                continue;
            }

            $source_post_type = sanitize_text_field($source_post_bits[0]);
            $source_post_id = (int)$source_post_bits[1];
            $source_data_type = (($source_post_type === 'post') ? 1: 0);
            $source_pid = $source_post_type . '_' . $source_post_id;
            if(!isset($ai_relation_data_cache[$source_pid])){
                $ai_relation_data_cache[$source_pid] = self::get_embedding_relatedness_data($source_post_id, $source_post_type);
            }

            $target_data = self::parse_linking_target($results);
            if($orphan_only_targets && !self::is_orphaned_target_for_ai_suggestion((int)$target_data['id'], (string)$target_data['type'])){
                continue;
            }

            $target_post = new Wpil_Model_Post((int)$target_data['id'], $target_data['type']);
            $target_pid = $target_post->get_pid();

            $ai_relation_score = 0;
            if(!empty($ai_relation_data_cache[$source_pid]) && !empty($target_pid) && isset($ai_relation_data_cache[$source_pid]->$target_pid) && !empty($ai_relation_data_cache[$source_pid]->$target_pid)){
                $ai_relation_score = (float)$ai_relation_data_cache[$source_pid]->$target_pid;
            }

            $sentence_text = wp_kses($results->sentence_text, 'post');
            $sentence_with_anchor_text = wp_kses($results->sentence_with_anchor_text, 'post');
            $sentence_id = (!empty($sentence_text)) ? md5($sentence_text) : '';

            array_push(
                $suggestion_data, 
                $source_post_id,
                $source_post_type,
                $source_data_type,
                $target_data['id'],
                $target_data['type'],
                $target_data['data_type'],
                $sentence_text,
                $sentence_id,
                $sentence_with_anchor_text,
                $ai_relation_score,
                (!empty($sentence_text) && !empty($sentence_with_anchor_text)) ? 0: 1,
                $process_key,
                time(),
                $model
            );

            $place_holders[] = "('%d', '%s', '%d', '%d', '%s', '%d', '%s', '%s', '%s', '%f', '%d', '%s', '%d', '%s')";

            // if we've hit the limit
            if($count > $limit || ($key + 1) >= $total){
                // assemble the insert
                $insert = ($insert_query . implode(', ', $place_holders));
                $insert = $wpdb->prepare($insert, $suggestion_data);
                // insert the data
                $wpdb->query($insert);
                // reset the data variables
                $suggestion_data = [];
                $place_holders = [];
                $count = 0;
            }

            $count++;
        }

        // if we still have data that hasn't been inserted
        if(!empty($suggestion_data) && !empty($place_holders)){
            // assemble the insert
            $insert = ($insert_query . implode(', ', $place_holders));
            $insert = $wpdb->prepare($insert, $suggestion_data);
            // and insert the data
            $wpdb->query($insert);
        }

        // return the total number of suggestions processed
        return $total_count;
    }

    public static function unwrap_linking_batch_item($item){
        $out = array(
            'ok'        => false,
            'model'     => '',
            'custom_id' => '',
            'results'   => array(),
        );

        if(empty($item)){
            return $out;
        }

        // Decode if it's a JSON string
        if(!is_object($item)){
            $item = self::decode($item);
        }

        if(empty($item) || !is_object($item)){
            return $out;
        }

        // Basic required wrapper fields
        if(
            !isset($item->response) ||
            !isset($item->response->status_code) ||
            (int)$item->response->status_code !== 200 ||
            !isset($item->response->body) ||
            !isset($item->response->body->choices) ||
            empty($item->response->body->choices) ||
            !isset($item->response->body->choices[0]) ||
            !isset($item->response->body->choices[0]->message) ||
            !isset($item->response->body->choices[0]->message->content)
        ){
            return $out;
        }

        $out['custom_id'] = isset($item->custom_id) ? (string)$item->custom_id : '';
        $out['model']     = isset($item->response->body->model) ? (string)$item->response->body->model : '';

        $content = $item->response->body->choices[0]->message->content;
        if(empty($content)){
            return $out;
        }

        // Content is JSON-as-string
        $decoded = self::decode($content);
        if(empty($decoded)){
            return $out;
        }

        // Normalize "results"
        $results = null;

        if(is_object($decoded) && isset($decoded->results)){
            $results = $decoded->results;
        } else {
            // allow raw array/object without results key
            $results = $decoded;
        }

        // If single object, wrap into array
        if(is_object($results)){
            $results = array($results);
        }

        if(!is_array($results) || empty($results)){
            return $out;
        }

        $out['ok'] = true;
        $out['results'] = $results;

        return $out;
    }

    public static function save_ai_outbound_linking_suggestions($live_response, $source_post = null, $process_key = ''){
        global $wpdb;
        $linking_table = $wpdb->prefix . 'wpil_ai_linking';

        // Source post fallback to the static origin post if it's set
        if(empty($source_post) && !empty(self::$origin_post)){
            $source_post = self::$origin_post;
        }

        if(empty($source_post) || !is_object($source_post) || empty($source_post->id)){
            return 0;
        }

        if(empty($live_response)){
            return 0;
        }

        $unwrapped = self::unwrap_linking_live_completion($live_response);
        if(!$unwrapped['ok']){
            return 0;
        }

        $results = $unwrapped['results'];
        $model   = $unwrapped['model'];

        $cols = array(
            'post_id',
            'post_type',
            'data_type',
            'target_id',
            'target_type',
            'target_data_type',
            'sentence_text',
            'sentence_id',
            'sentence_with_anchor_text',
            'ai_relation_score',
            'ignored',
            'process_time',
            'model_version',
            'process_key'
        );

        $insert_query = "INSERT INTO {$linking_table} (" . implode(',', $cols) . ") VALUES ";

        $suggestion_data = array();
        $place_holders   = array();

        $count = 0;
        $limit = 1000;

        $source_post_id   = (int) $source_post->id;
        $source_post_type = !empty($source_post->type) ? (string)$source_post->type : 'post';
        $data_type        = (($source_post_type === 'post') ? 1 : 0);

        // Get AI relation data for the source post
        $ai_relation_data = self::get_embedding_relatedness_data($source_post->id, $source_post->type);

        foreach($results as $r){
            $target_data = self::parse_linking_target($r);
            $target_id = (int) $target_data['id'];
            $target_type = $target_data['type'];

            $sentence_text = wp_kses((string)$r->sentence_text, 'post');
            $sentence_id = (!empty($sentence_text)) ? md5($sentence_text) : '';
            $sentence_with = wp_kses((string)$r->sentence_with_anchor_text, 'post');

            // Calculate ai_relation_score the same way as insert_links_into_link_table
            $ai_relation_score = 0;
            if(!empty($ai_relation_data)){
                $target_post = new Wpil_Model_Post($target_id, $target_type);
                $pid = $target_post->get_pid();
                if(isset($ai_relation_data->$pid) && !empty($ai_relation_data->$pid)){
                    $ai_relation_score = $ai_relation_data->$pid;
                }
            }

            $ignored = (!empty($sentence_text) && !empty($sentence_with)) ? 0 : 1;

            array_push(
                $suggestion_data,
                $source_post_id,
                $source_post_type,
                $data_type,
                $target_id,
                $target_type,
                $target_data['data_type'],
                $sentence_text,
                $sentence_id,
                $sentence_with,
                $ai_relation_score,
                $ignored,
                time(),
                $model,
                $process_key
            );

            $place_holders[] = "('%d','%s','%d','%d','%s','%d','%s','%s','%s','%f','%d','%d','%s','%s')";
            $count++;

            // Flush chunk
            if($count >= $limit){
                $insert = $insert_query . implode(', ', $place_holders);
                $insert = $wpdb->prepare($insert, $suggestion_data);
                $wpdb->query($insert);

                $suggestion_data = array();
                $place_holders   = array();
                $count = 0;
            }
        }

        // Final flush!
        if(!empty($suggestion_data) && !empty($place_holders)){
            $insert = $insert_query . implode(', ', $place_holders);
            $insert = $wpdb->prepare($insert, $suggestion_data);
            $wpdb->query($insert);
        }

        return count($results);
    }

    /**
     * Unwraps for the OUTBOUND suggestions
     **/
    public static function unwrap_linking_live_completion($item){
        $out = array(
            'ok'      => false,
            'model'   => '',
            'results' => array(),
        );

        if(empty($item)){
            return $out;
        }

        // Decode if it's a JSON string
        if(!is_object($item)){
            $item = self::decode($item);
        }

        if(empty($item) || !is_object($item)){
            return $out;
        }

        // Live chat completion shape: choices[0].message.content
        if(
            !isset($item->choices) ||
            empty($item->choices) ||
            !isset($item->choices[0]) ||
            !isset($item->choices[0]->message) ||
            !isset($item->choices[0]->message->content)
        ){
            return $out;
        }

        $out['model'] = isset($item->model) ? (string)$item->model : '';

        $content = $item->choices[0]->message->content;
        if(empty($content) || !is_string($content)){
            return $out;
        }

        // content is JSON-as-string: {"results":[...]}
        $decoded = self::decode($content);
        if(empty($decoded)){
            return $out;
        }

        $results = null;

        if(is_object($decoded) && isset($decoded->results)){
            $results = $decoded->results;
        } else {
            // Allow raw array/object without results key
            $results = $decoded;
        }

        if(is_object($results)){
            $results = array($results);
        }

        if(!is_array($results) || empty($results)){
            return $out;
        }

        // Validate basic required fields for each item
        $clean = array();
        foreach($results as $r){
            if(!is_object($r)){
                continue;
            }
            if(!isset($r->sentence_text) || !isset($r->sentence_with_anchor_text) || !isset($r->target_id)){
                continue;
            }
            $clean[] = $r;
        }

        if(empty($clean)){
            return $out;
        }

        $out['ok'] = true;
        $out['results'] = $clean;

        return $out;
    }

    /**
     * Gets the ids for all the suggested sentences that we've processed with AI for this post
     * @param Wpil_Model_Post $post
     * @param bool $reindex Should we reindex the results so we can quickly search for a specific id using isset?
     **/
    public static function get_post_suggestion_anchor_ids($post = array(), $reindex = false){
        global $wpdb;
        $anchor_table = $wpdb->prefix . 'wpil_ai_suggested_anchors';

        if(empty($post) || !is_a($post, 'Wpil_Model_Post')){
            return false;
        }

        $ids = $wpdb->get_results($wpdb->prepare("SELECT `sentence_post_id` FROM {$anchor_table} WHERE `post_id` = %s AND `post_type` = %d", $post->id, $post->type));
    
        if(!empty($ids) && $reindex){
            $reindexed = array();
            foreach($ids as $id){
                $reindexed[$id->sentence_post_id] = true;
            }
            $ids = $reindexed;
        }

        return $ids;
    }

    /**
     * Gets the AI processed sentences that are good enough to be viable
     * @param Wpil_Model_Post $post
     **/
    public static function get_ai_post_suggestion_sentences($post = array(), $decode = true){
        global $wpdb;
        $anchor_table = $wpdb->prefix . 'wpil_ai_suggested_anchors';

        if(empty($post) || !is_a($post, 'Wpil_Model_Post')){
            return false;
        }

        $suggestion_data = $wpdb->get_results($wpdb->prepare("SELECT `post_id`, `post_type`, `sentence_post_id`, `sentence_id`, `suggestion_words`, `notes`, `target_id`, `target_type` FROM {$anchor_table} WHERE `post_id` = %s AND `post_type` = %d AND `ignore_suggestion` = 0", $post->id, $post->type));
        $suggestions = array();
        if(!empty($suggestion_data)){
            foreach($suggestion_data as $dat){
                if(!isset($suggestions[$dat->sentence_id])){
                    $suggestions[$dat->sentence_id] = array();
                }

                if($decode){
                    $dat->suggestion_words = Wpil_Toolbox::json_decompress($dat->suggestion_words);
                    if(!empty($dat->notes)){
                        $dat->notes = Wpil_Toolbox::json_decompress($dat->notes);
                    }
                }

                $suggestions[$dat->sentence_id][] = $dat;
            }
        }

        return $suggestions;
    }

    /**
     * Deletes the phrase embedding data for a specific post
     * @param Wpil_Model_Post $post
     **/
    public static function clear_post_phrase_suggestion_sentences($post = array(), $age = 0){
        global $wpdb;
        $table = $wpdb->prefix . "wpil_ai_suggested_anchors";

        if(empty($post) || !is_a($post, 'Wpil_Model_Post')){
            return true;
        }

        $where = "`post_id` = {$post->id} AND `post_type` = '{$post->type}'";

        if(!empty($age) && is_numeric($age)){
            $age = intval($age);
            $where .= " AND `process_time` < {$age}";
        }

        $wpdb->query("DELETE FROM {$table} WHERE $where");
    }

    /**
     * Inserts processed sentences into the processed anchor sentence table
     * @param string $sentence The sentence that we're planning to log
     * @param Wpil_Model_Post The post object that the sentence belongs to
     **/
    public static function log_processed_anchor_sentence($sentence = '', $post = array()){
        global $wpdb;
        $table = $wpdb->prefix . "wpil_ai_processed_sentences";

        if(empty($post) || !is_a($post, 'Wpil_Model_Post') || empty($sentence)){
            return false;
        }

        if(preg_match('/^[a-f0-9]{32}$/i', $sentence)){
            $sentence_id = $sentence;
        }else{
            $sentence_id = md5($sentence);
        }

        $has_link = false;
        $phrase = Wpil_Suggestion::getPhrasebyId($sentence_id, $post);
        if(!empty($phrase)){
            $has_link = (Wpil_Link::hasLink($phrase->sentence_src));
        }

        $sentences = self::get_processed_anchor_sentences($post);
        if(!in_array($sentence, $sentences)){
            $wpdb->insert($table, array(
                'post_id' => $post->id,
                'post_type' => $post->type,
                'data_type' => (($post->type === 'post') ? 1: 0),
                'has_link' => (($has_link) ? 1: 0),
                'sentence_id' => $sentence_id,
                'process_time' => time()
            ));

            if(!isset(self::$sentence_anchor_cache[$post->get_pid()])){
                self::$sentence_anchor_cache[$post->get_pid()] = array();
            }
            self::$sentence_anchor_cache[$post->get_pid()][] = $sentence;
        }
    }

    /**
     * Gets processed sentence references from the processed anchor sentence table
     * @param Wpil_Model_Post The post object that the sentence belongs to
     **/
    public static function get_processed_anchor_sentences($post = array(), $reindex = false, $ignore_cache = false){
        if(empty($post) || !is_a($post, 'Wpil_Model_Post')){
            return false;
        }

        if(!isset(self::$sentence_anchor_cache[$post->get_pid()]) || $ignore_cache){
            self::load_processed_anchor_sentence_cache($post);
        }

        $ids = self::$sentence_anchor_cache[$post->get_pid()];

        if($reindex && !empty($ids)){
            $reindexed = array();
            foreach($ids as $id){
                $reindexed[$id] = true;
            }
            $ids = $reindexed;
        }

        return $ids;
    }

    /**
     * Loads the sentence cache so we don't have to hit the database every time we want to check a sentence
     * @param Wpil_Model_Post The post object that we're pulling sentences from
     **/
    public static function load_processed_anchor_sentence_cache($post = array()){
        global $wpdb;
        $table = $wpdb->prefix . "wpil_ai_processed_sentences";

        if(empty($post) || !is_a($post, 'Wpil_Model_Post')){
            return false;
        }

        $sentences = $wpdb->get_col($wpdb->prepare("SELECT `sentence_id` FROM {$table} WHERE `post_id` = %d AND `post_type` = %s", $post->id, $post->type));
        $processed = array();
        if(!empty($sentences)){
            foreach($sentences as $sentence){
                $processed[$sentence] = true;
            }

            if(!empty($processed)){
                $processed = array_keys($processed);
            }
        }
        
        self::$sentence_anchor_cache[$post->get_pid()] = $processed;
    }


    /**
     * Clears processed sentence data from the processed anchor sentence table
     * @param Wpil_Model_Post The post object that the sentence belongs to
     **/
    public static function clear_processed_anchor_sentences($post = array(), $age = 0){
        global $wpdb;
        $table = $wpdb->prefix . "wpil_ai_processed_sentences";

        if(empty($post) || !is_a($post, 'Wpil_Model_Post')){
            return false;
        }

        $where = "`post_id` = {$post->id} AND `post_type` = '{$post->type}'";

        if(!empty($age) && is_numeric($age)){
            $age = intval($age);
            $where .= " AND `process_time` < {$age}";
        }

        $wpdb->query("DELETE FROM {$table} WHERE $where");
    }

    /**
     * Gets the post ids for all posts that are currently inserted in one of the databases that relys on OIA batch data.
     * The "ids" are a combination of 'post->type' . '_' . 'post->id' so they can be easily compared to the "custom_id" that was set for the batch item
     * 
     * @param string $database What database should we be checking for posts?
     * @return array The list of all the posts that are currently stored in the database.
     **/
    public static function get_inserted_post_data($database = ''){
        global $wpdb;
        $summarized_posts = $wpdb->prefix . 'wpil_ai_post_data';
        $embedding_data = $wpdb->prefix . 'wpil_ai_embedding_data';
        $product_data = $wpdb->prefix . 'wpil_ai_product_data';
        $keyword_data = $wpdb->prefix . 'wpil_ai_keyword_data';
        $post_ids = array();

        if(empty($database)){
            return $post_ids;
        }

        if($database === 'post-summarizing'){
            $results = $wpdb->get_results("SELECT `post_id`, `post_type` FROM {$summarized_posts}");
        }elseif($database === 'create-post-embeddings'){
            $results = $wpdb->get_results("SELECT `post_id`, `post_type` FROM {$embedding_data}");
        }elseif($database === 'product-detecting'){
            $results = $wpdb->get_results("SELECT `post_id`, `post_type` FROM {$product_data}");
        }elseif($database === 'keyword-detecting'){
            $results = $wpdb->get_results("SELECT `post_id`, `post_type` FROM {$keyword_data}");
        }elseif($database === 'summary-and-product-searching'){
            $results = $wpdb->get_results("SELECT b.post_id, b.post_type FROM {$summarized_posts} a LEFT JOIN {$product_data} b ON a.post_id = b.post_id AND a.data_type = b.data_type WHERE b.post_id > 0");
        }elseif($database === 'summary-and-keyword-searching'){
            $results = $wpdb->get_results("SELECT b.post_id, b.post_type FROM {$summarized_posts} a LEFT JOIN {$keyword_data} b ON a.post_id = b.post_id AND a.data_type = b.data_type WHERE b.post_id > 0");
        }elseif($database === 'product-and-keyword-searching'){
            $results = $wpdb->get_results("SELECT b.post_id, b.post_type FROM {$product_data} a LEFT JOIN {$keyword_data} b ON a.post_id = b.post_id AND a.data_type = b.data_type WHERE b.post_id > 0");
        }elseif($database === 'summary-keyword-and-product-searching'){
            $results = $wpdb->get_results(
                "SELECT b.post_id, b.post_type FROM {$summarized_posts} a 
                    LEFT JOIN {$keyword_data} b ON a.post_id = b.post_id AND a.data_type = b.data_type 
                    LEFT JOIN {$product_data} c ON b.post_id = c.post_id AND b.data_type = c.data_type 
                    WHERE c.post_id > 0");
        }

        if(!empty($results)){
            foreach($results as $result){
                $id = $result->post_type . '_' . $result->post_id;
                $post_ids[$id] = true;
            }
        }

        return $post_ids;
    }

    /**
     * Compiles the batch process file and shoots it at OAI
     **/
    public static function start_batch_process($data, $instructions = '', $endpoint = ''){
        if(empty($data) || empty($instructions) || empty(self::$ai)){
            return false;
        }

        $time = time();
        $file = self::get_upload_file($time);
        $filesize = 0;
        $file_limit = 50000000;
        $site_id = md5(get_site_url());

        if(empty($file)){
            return false;
        }

        switch ($endpoint) {
            case 'embeddings':
                $url = '/v1/embeddings';
                $base_data = array(
                    'custom_id' => null,
                    'method' => 'POST',
                    'url' => $url,
                    'body' => array(
                        'model' => self::$model,
                        'input' => '',
                        'dimensions' => Wpil_Settings::get_ai_dimension_limit()
                    )
                );
                break;
            case 'completions':
            default:
                $url = '/v1/chat/completions';
                $base_data = array(
                    'custom_id' => null,
                    'method' => 'POST',
                    'url' => $url,
                    'body' => array(
                        'model' => self::$model,
                        'messages' => array(
                            array(
                                'role' => 'system',
                                'content' => $instructions
                            )
                        ),
                        'response_format' => self::create_response_format('batch_process', self::create_response_indexes(self::$purpose), self::$model),
                    )
                );
                break;
        }

        $first = true;
        foreach($data as $key => $dat){
            // if there's no content
            if(empty($dat)){
                // save the empty data in the tables
                self::save_empty_post_data($key, self::$model);
                // remove the empty index
                unset($data[$key]);
                // skip to the next
                continue;
            }

            $d = $base_data;
            $d['custom_id'] = $key;
            if($endpoint === 'embeddings'){
                $d['body']['input'] = $dat;
            }else{
                $d['body']['messages'][] = array('role' => 'user', 'content' => $dat);
            }
            $d = json_encode($d);
            if(!empty($d) && is_string($d)){
                $len = strlen($d);
                if($filesize + $len > $file_limit){
                    break;
                }else{
                    $filesize += $len;
                }

                if(!$first){
                    $d = "\n" . $d;
                }else{
                    $first = false;
                }
                fwrite($file, $d);
            }
        }

        $stat = fstat($file);
        if(!empty($stat) && $stat['size'] > 0){
            fclose($file);
            $response = self::decode(self::$ai->uploadFile(
                array(
                    'purpose' => 'batch',
                    'file' => curl_file_create(self::get_upload_dir() . 'OAI_' . $time . '_upload.jsonl'),
                )
            ));

            // clear the upload file now that we're done with it
            self::delete_upload_files();

            if(isset($response->error) && !empty($response->error)){
                self::save_system_error_log_data($response);
                if(
                    isset($response->error->code) && 
                    !empty($response->error->code) && 
                    ($response->error->code === 'insufficient_quota' || $response->error->code === 'billing_hard_limit_reached'))
                {
                    update_option('wpil_oai_insufficient_quota_error', '1');
                }
            }

            self::save_response_tokens($response, self::$purpose, true);

            if(!empty($response) && isset($response->id) && !empty($response->id)){
                $response = self::decode(self::$ai->createBatch(
                    array(
                        'input_file_id' => $response->id,
                        'endpoint' => $url,
                        'completion_window' => '24h',
                        'metadata' => array(
                            'purpose' => self::$purpose,
                            'site_id' => $site_id
                        )
                    )
                ));

                if(isset($response->error) && !empty($response->error)){
                    self::save_system_error_log_data($response);
                    if(
                        isset($response->error->code) && 
                        !empty($response->error->code) && 
                        ($response->error->code === 'insufficient_quota' || $response->error->code === 'billing_hard_limit_reached'))
                    {
                        update_option('wpil_oai_insufficient_quota_error', '1');
                    }
                }

                if(!empty($response) && isset($response->id)){
                    $batches = get_option('wpil_oai_batch_data', array());
                    $batches[] = $response;
                    update_option('wpil_oai_batch_data', $batches);
                    // save the data that we're processing so we don't queue up duplicates
                    self::save_batch_log_data($response->id, array_keys($data), self::get_process_code_from_name(self::$purpose));
                }
            }
        }
    }

    /**
     * Checks to see how the current batch is doing
     **/
    public static function check_batch_process($batch_id = ''){
        if(empty($batch_id) || !is_string($batch_id)){
            return false;
        }

        $batch = self::decode(self::$ai->retrieveBatch($batch_id));
        $status = 'running';
        if(!empty($batch) && isset($batch->status)){
            switch ($batch->status) {
                case 'failed':
                case 'expired':
                case 'cancelled':
                case 'cancelling':
                    self::delete_batch($batch);
                    $status = 'deleted';
                    break;
                case 'completed':
                    self::mark_batch_complete($batch);
                    $status = 'completed';
                    break;
                default:
            }
        }elseif(!empty($batch) && isset($batch->error) && !empty($batch->error)){
            // if there's an error, assume it's because the batch doesn't exist
            $status = 'deleted';
        }

        return $status;
    }

    /**
     * Adds a completed batch to the list of batches that are ready for processing
     * @param object $batch
     **/
    public static function mark_batch_complete($batch = array()){
        if(empty($batch)){
            return false;
        }

        $batches = get_option('wpil_oai_completed_batch_data', array());
        $listed = false;
        if(!empty($batches)){
            foreach($batches as $key => $dat){
                if(!isset($dat->id) || empty($dat->id)){
                    unset($batches[$key]);
                    continue;
                }

                // if the batch is already logged
                if($dat->id === $batch->id){
                    // replace it since this is more recent
                    $batches[$key] = $batch;
                    // and make a note of it
                    $listed = true;
                }
            }
        }

        // if the batch isn't already logged
        if(!$listed){
            // add it to the complete list
            $batches[] = $batch;
        }
        
        update_option('wpil_oai_completed_batch_data', $batches);

        return true;
    }

    /**
     * Deletes a batch from the OpenAI storage and removes the listing from our cache
     * @param object $batch
     **/
    public static function delete_batch($batch = array()){
        if(empty($batch)){
            return false;
        }

        // temp removing so I can pull down the file!
        if(isset($batch->input_file_id) && !empty($batch->input_file_id)){
            self::$ai->deleteFile($batch->input_file_id);
        }

        if(isset($batch->output_file_id) && !empty($batch->output_file_id)){
            self::$ai->deleteFile($batch->output_file_id);
        }

        if(isset($batch->error_file_id) && !empty($batch->error_file_id)){
            self::$ai->deleteFile($batch->error_file_id);
        }

        if( isset($batch->errors) && !empty($batch->errors) && 
            isset($batch->errors->data) && !empty($batch->errors->data) && 
            isset($batch->metadata) && !empty($batch->metadata) &&
            isset($batch->metadata->purpose) && !empty($batch->metadata->purpose) &&
            is_array($batch->errors->data))
        {
            foreach($batch->errors->data as $dat){
                $delayed = false;
                if(!empty($dat) && isset($dat->code) && !empty($dat->code) && $dat->code === 'token_limit_exceeded'){
                    $delayed = self::delay_batch_process($batch->metadata->purpose);
                }

                if(isset($dat->message) && !empty($dat->message) && empty($delayed)){
                    self::save_system_error_log_data($batch, $dat->message);
                    if(
                        isset($response->error->code) && 
                        !empty($response->error->code) && 
                        ($response->error->code === 'insufficient_quota' || $response->error->code === 'billing_hard_limit_reached'))
                    {
                        update_option('wpil_oai_insufficient_quota_error', '1');
                    }
                }
            }
        }

        self::save_completed_batch_log_entry($batch->id, $batch->status);
        self::delete_batch_log_data($batch->id);
        if(isset($batch->metadata, $batch->metadata->purpose) && !empty($batch->metadata->purpose)){
            self::clear_credit_tracking_task_run('purpose:' . $batch->metadata->purpose);
        }

        $batch_cache = get_option('wpil_oai_batch_data', array());
        $batch_completed = get_option('wpil_oai_completed_batch_data', array());

        if(!empty($batch_cache)){
            foreach($batch_cache as $key => $dat){
                if(isset($dat->id, $batch->id) && $dat->id === $batch->id){
                    unset($batch_cache[$key]);
                }
            }

            update_option('wpil_oai_batch_data', $batch_cache);
        }

        if(!empty($batch_completed)){
            foreach($batch_completed as $key => $dat){
                if(isset($dat->id, $batch->id) && $dat->id === $batch->id){
                    unset($batch_completed[$key]);
                }
            }
            update_option('wpil_oai_completed_batch_data', $batch_completed);
        }
    }

    /**
     * Gets the data from a completed batch process
     **/
    public static function get_batch_data($file_id = ''){
        if(empty($file_id) || !is_string($file_id)){
            return false;
        }

        // retrieve the json data. Is JSONL, so we don't need to decode it
        $file = self::$ai->retrieveFileContent($file_id);
        $decoded = self::decode($file, true); // in fact, if we decode it, there should be an error
        if(!empty($file) && empty($decoded)){
            $file = explode("\n", $file);
            if(!empty($file)){
                return array_filter(array_map('trim', $file));
            }
        }elseif(!empty($file) && !empty($decoded) && isset($decoded->error)){
            return $decoded;
        }elseif(!empty($file) && !empty($decoded) && isset($decoded->id, $decoded->custom_id, $decoded->response) && !empty($decoded->id) && !empty($decoded->custom_id) && !empty($decoded->response) && is_object($decoded)){
            // if there appears to be only one item, wrap it in an array so we can process
            return array($decoded);
        }

        return false;
    }

    /**
     * Processes the data from a completed batch.
     * Automatically selects and manages the completed batches, and cleans up completed && cancelled batches
     **/
    public static function process_batch_data(){
        if(empty(self::$ai)){
            return;
        }

        $batch_completed = get_option('wpil_oai_completed_batch_data', []);
        $site_id = md5(get_site_url());

        // if there are no batches completed
        if(empty($batch_completed)){
            // check and see if there are batches logged
            $batches = get_option('wpil_oai_batch_data', array());
            
            // make sure that we have all the logged ids
//            $batch_log_ids = self::get_batch_log_ids();
/*
            if(!empty($batch_log_ids)){
                foreach($batch_log_ids as $log_id){
                    if(!empty($batches)){
                        foreach($batches as $key => $batch){
                            if(!isset($batch->id) || empty($batch->id)){
                                unset($batches[$key]);
                                continue;
                            }
        
                            // if the batch id is logged in the option
                            if($batch->id === $log_id){
                                // skip it since we don't need to process it
                                continue 2;
                            }
                        }
                    }

                    $batch = self::decode(self::$ai->retrieveBatch($log_id));
                    if(!empty($batch)){
                        $batches[] = $batch;
                    }
                }
            }*/

            $stored_batches = self::decode(self::$ai->listBatches());

            if(!empty($stored_batches) && isset($stored_batches->data)){
                foreach($stored_batches->data as $stored_batch){
                    if( !isset($stored_batch->id) || 
                        empty($stored_batch->id) || 
                        !empty(self::get_completed_batch_log_entries($stored_batch->id)) ||
                        isset($stored_batch->expires_at) && !empty($stored_batch->expires_at) && ($stored_batch->expires_at + WEEK_IN_SECONDS < time())
                    ){
                        continue;
                    }

                    if( isset($stored_batch->metadata, $stored_batch->metadata->site_id) && 
                        !empty($stored_batch->metadata) && 
                        !empty($stored_batch->metadata->site_id) &&
                        trim($stored_batch->metadata->site_id) !== $site_id)
                    {
                        self::save_completed_batch_log_entry($stored_batch->id, 'not_this_site');
                        continue;
                    }

                    if(!empty($batches)){
                        foreach($batches as $key => $batch){
                            if(!isset($batch->id) || empty($batch->id)){
                                unset($batches[$key]);
                                continue;
                            }
            
                            // if the batch id is logged in the option
                            if($batch->id === $stored_batch->id){
                                // skip it since we don't need to process it
                                continue 2;
                            }
                        }
                    }
            
                    $batches[] = $stored_batch;
                }
            }

            // if there are
            if(!empty($batches)){
                foreach($batches as $key => $batch){
                    if(!isset($batch->id) || empty($batch->id)){
                        unset($batches[$key]);
                        continue;
                    }

                    $process = self::check_batch_process($batch->id);
                    if($process === 'deleted' || $process === 'completed'){
                        unset($batches[$key]);
                    }
                }

                update_option('wpil_oai_batch_data', $batches);
            }

            return false;
        }

        $saved = 0;
        foreach($batch_completed as $key => $batch){
            if( !isset($batch->id) || empty($batch->id) || 
                !isset($batch->output_file_id) || empty($batch->output_file_id) ||
                !isset($batch->metadata) || empty($batch->metadata) ||
                !empty(self::get_completed_batch_log_entries($batch->id))
            ){
                if(isset($batch->error_file_id) && !empty($batch->error_file_id)){
                    if(isset($batch->id)){
                        self::save_error_log_data($batch->id, self::get_batch_data($batch->error_file_id));
                    }
                }

                unset($batch_completed[$key]);
                self::delete_batch($batch);
                continue;
            }

            $data = self::get_batch_data($batch->output_file_id);

            // if we got an error when requesting the batch data
            if(!empty($data) && isset($data->error)){
                // remove the batch
                // Known errors:
                // invalid_request_error => happens when there's no file to download. (previuosly deleted or expired)
                self::delete_batch($batch);
                continue;
            }

            if(!empty($data) && is_array($data)){
                // check the meta to determine what to do with it
                if($batch->metadata->purpose === 'create-post-embeddings'){
                    // if we're doing embeddings
                    // save the embedding data
                    $saved = self::save_site_embeddings($data);
                }elseif($batch->metadata->purpose === 'post-summarizing'){
                    $saved = self::save_site_post_summaries($data);
                }elseif($batch->metadata->purpose === 'product-detecting'){
                    $saved = self::save_site_products($data);
                }elseif($batch->metadata->purpose === 'keyword-detecting'){
                    $saved = self::save_site_keywords($data);
                }elseif($batch->metadata->purpose === 'summary-and-product-searching'){
                    $saved1 = self::save_site_post_summaries($data);
                    $saved2 = self::save_site_products($data);
                    $saved = max([$saved1, $saved2]);

                    // todo: create delete_after_insert function that will use a DB command to find duplicate items and delete the oldest ones
                }elseif($batch->metadata->purpose === 'summary-and-keyword-searching'){
                    $saved1 = self::save_site_post_summaries($data);
                    $saved2 = self::save_site_keywords($data);
                    $saved = max([$saved1, $saved2]);

                    // todo: create delete_after_insert function that will use a DB command to find duplicate items and delete the oldest ones
                }elseif($batch->metadata->purpose === 'product-and-keyword-searching'){
                    $saved1 = self::save_site_products($data);
                    $saved2 = self::save_site_keywords($data);
                    $saved = max([$saved1, $saved2]);
                    // todo: create delete_after_insert function that will use a DB command to find duplicate items and delete the oldest ones
                }elseif($batch->metadata->purpose === 'summary-keyword-and-product-searching'){
                    

                    $saved1 = self::save_site_post_summaries($data);
                    $saved2 = self::save_site_keywords($data);
                    $saved3 = self::save_site_products($data);

                    $saved = max([$saved1, $saved2, $saved3]);

                    // todo: create delete_after_insert function that will use a DB command to find duplicate items and delete the oldest ones
                }else{
                    // remove the batch if it's not on the list
                    self::delete_batch($batch);
                }

                if(isset($batch->error_file_id) && !empty($batch->error_file_id)){
                    // save any errors if present
                    self::save_error_log_data($batch->id, self::get_batch_data($batch->error_file_id));
                }

                if(!empty($saved)){
                    self::save_response_tokens($data, $batch->metadata->purpose, true);
                }

                // if all the items in the data are saved
                if($saved == count($data)){
                    // remove the completed batch!
                    self::delete_batch($batch);
                    // and break out of the loop
                    break;
                }
            }
        }
    }

    /**
     * 
     **/
    public static function live_query_ai_results($data, $instructions = '', $endpoint = ''){
        if(empty($data) || empty($instructions) || empty(self::$ai)){
            return false;
        }

        if($endpoint === 'embeddings'){
            $dimensions = Wpil_Settings::get_ai_dimension_limit();
            $message_list = array();
            self::$query_ids = array();
            foreach($data as $post_id => $dat){
                if(is_array($dat)){ // we really shouldn't be chunking embedding data...
                    foreach($dat as $chunk_data){
                        $message_list[] = array(
                            "model" => self::$model,
                            "input" => $chunk_data,
                            "dimensions" => $dimensions
                        );
                        self::$query_ids[] = $post_id;
                    }
                }else{
                    $message_list[] = array(
                        "model" => self::$model,
                        "input" => $dat,
                        "dimensions" => $dimensions
                    );
                    self::$query_ids[] = $post_id;
                }
            }

            $args = array(
                'message_list' => $message_list,
            );

            $results = self::$ai->embeddings($args, true);

            if(!empty($results)){
                foreach($results as $key => $dat){
                    
                    $response = self::decode($dat);
                    if( !empty($response) &&
                        ((isset($response->error) && !empty($response->error)) ||
                        (isset($response->statusCode) && ($response->statusCode > 203 || $response->statusCode < 200)))
                    ){
                        if(isset($response->error)){
                            if(isset($response->error->message) && !empty($response->error->message)){
                                self::$error_message = esc_html($response->error->message);
                            }

                            if(isset($response->error->type)){
                                if($response->error->type === 'invalid_request_error'){
                                    self::$invalid_request = true;
                                    self::track_error($response->error->type);
                                }
                            }

                            if(isset($response->error->code)){
                                if($response->error->code === 'rate_limit_exceeded'){
                                    self::$rate_limited = true;
                                }elseif($response->error->code === 'invalid_prompt'){
            
                                }elseif($response->error->code === 'insufficient_quota' || $response->error->code === 'billing_hard_limit_reached'){
                                    self::$insufficient_quota = true;
                                }elseif($response->error->code === 'invalid_api_key'){
                                    self::$invalid_api_key = true;
                                }
                                
                                self::track_error($response->error->code);
                            }
                        }elseif(self::$ai_service_connected){
                            if(isset($response->body)){
                                if(is_string($response->body)){
                                    $response->body = json_decode($response->body);
                                }

                                if(isset($response->body->error)){
                                    if(isset($response->body->error)){
                                        if($response->body->error === 'User not found'){
                                            self::$user_not_exist = true;
                                        }elseif($response->body->error === 'Insufficient credits'){
                                            self::$insufficient_quota = true;
                                        }elseif($response->body->error === 'Access not valid'){
                                            self::$invalid_api_key = true;
                                        }
                                        
                                        self::track_error($response->body->error);
                                    }
                                }
                            }
                        }

                        // format the data for saving
                        $dat_object = (object)array(
                            'response' => array(
                                'status_code' => 200,
                                'body' => $response,
                                'purpose' => self::$purpose
                            ),
                            'custom_id' => self::$query_ids[$key],
                            'id' => 'live_download'
                        );
    
                        $dat_object = json_encode($dat_object);
                        if(!empty($dat_object)){
                            $dat_object = array($dat_object);
                            self::save_error_log_data('live_download', $dat_object, self::$purpose);
                            // and mark the post as processed if there isn't a temp/quota error
                            if(!self::$insufficient_quota && !self::$rate_limited && !self::$invalid_request && !self::$invalid_api_key && !self::$user_not_exist){
                                self::save_empty_post_data(self::$query_ids[$key], self::$model, self::$purpose);
                            }

                            if(self::$invalid_api_key || self::$user_not_exist){
                                return;
                            }else{
                                continue;
                            }
                        }
                    }
        
                    $dat_object = (object)array(
                        'response' => array(
                            'status_code' => 200,
                            'body' => $response
                        ),
                        'custom_id' => self::$query_ids[$key],
                    );
        
                    $dat_object = json_encode($dat_object);
                    
                    if(!empty($dat_object)){
                        $dat_object = array($dat_object);
                        self::save_site_embeddings($dat_object);
                    }
        
                    self::save_response_tokens($dat, self::$purpose, false, isset(self::$query_ids[$key]) ? self::$query_ids[$key] : '');
                }
            }
        }else{
            $message_list = array();
            self::$query_ids = array();
            foreach($data as $post_id => $dat){
                if(is_array($dat)){
                    foreach($dat as $chunk_data){
                        $message_list[] = 
                            ['messages' => [
                                [
                                    "role" => "system",
                                    "content" => $instructions
                                ],
                                [
                                    "role" => "user",
                                    "content" => $chunk_data
                                ]
                                ]
                            ];
                        self::$query_ids[] = $post_id;
                        self::$chunked_posts[$post_id] = true;
                    }
                }else{
                    $message_list[] = 
                        ['messages' => [
                            [
                                "role" => "system",
                                "content" => $instructions
                            ],
                            [
                                "role" => "user",
                                "content" => $dat
                            ]
                            ]
                        ];
                    self::$query_ids[] = $post_id;
                }
            }

            $args = array(
                'model' => self::$model,
                'response_format' => self::create_response_format('live_download', self::create_response_indexes(self::$purpose), self::$model, (self::$purpose === 'assess-sentence-anchors')),
                'message_list' => $message_list,
                'temperature' => 0,
                //'top_p' => 0.1,
                'frequency_penalty' => 0,
                'presence_penalty' => -2.0
            );

            $chat = self::$ai->chat($args, null, true);
            $merge_data = array();
            foreach($chat as $chat_id => $dat){
                // if there was no response but we did have content to supply
                if(empty($dat) && isset($args['message_list'][$chat_id]) && !empty($args['message_list'][$chat_id])){
                    // if we've already chunked the post
                    if(isset(self::$chunked_posts[self::$query_ids[$chat_id]])){
                        // remove it from the chunked post list
                        self::remove_chunked_post(self::$query_ids[$chat_id]);
                        // save the empty data
                        self::save_empty_post_data(self::$query_ids[$chat_id], self::$model, self::$purpose);
                        // and continue
                        continue;
                    }

                    // assume that we need to chunck the request to get it past OAI
                    self::save_chunked_post(self::$query_ids[$chat_id]);
                    continue;
                }

                $response = self::decode($dat);

                if(!empty($response) && isset(self::$chunked_posts[self::$query_ids[$chat_id]])){
                    $next = $chat_id + 1;
                    $sub_dat = self::decode($response->choices[0]->message->content);

                    // add the response content to the chunk merge data
                    if(!isset($merge_data[self::$query_ids[$chat_id]])){
                        $merge_data[self::$query_ids[$chat_id]] = array(
                            'keywords' => array(),
                            'keyword-count' => 0,
                            'products' => array(),
                            'product-count' => 0
                        );
                    }

                    if(!empty($sub_dat) && isset($sub_dat->results)){
                        if(isset($sub_dat->results->keywords)){ 
                            $merge_data[self::$query_ids[$chat_id]]['keywords'] = array_unique(array_merge($merge_data[self::$query_ids[$chat_id]]['keywords'], explode(',', $sub_dat->results->keywords)));
                            $merge_data[self::$query_ids[$chat_id]]['keyword-count'] = count($merge_data[self::$query_ids[$chat_id]]['keywords']);
                        }
                        if(isset($sub_dat->results->products)){ 
                            $merge_data[self::$query_ids[$chat_id]]['products'] = array_unique(array_merge($merge_data[self::$query_ids[$chat_id]]['products'], explode(',', $sub_dat->results->products)));
                            $merge_data[self::$query_ids[$chat_id]]['product-count'] = count($merge_data[self::$query_ids[$chat_id]]['products']);
                        }
                    }

                    if(
                        isset($chat[$next]) && // if there's another item in the chat
                        isset(self::$query_ids[$next]) && // and we have an id for it
                        isset(self::$chunked_posts[self::$query_ids[$next]]) && // and the next item is chunked
                        self::$query_ids[$chat_id] === self::$query_ids[$next] // and the id is the same as this one
                    ){
                        // note the tokens used for this request
                        self::save_response_tokens($dat, self::$purpose);
                        // and continue on to the next item
                        continue;
                    }

                    // if this is the last item, update the response content
                    $merge_data[self::$query_ids[$chat_id]]['keywords'] = implode(',', $merge_data[self::$query_ids[$chat_id]]['keywords']);
                    $merge_data[self::$query_ids[$chat_id]]['products'] = implode(',', $merge_data[self::$query_ids[$chat_id]]['products']);
                    $response->choices[0]->message->content = json_encode(array('results' => $merge_data[self::$query_ids[$chat_id]]));
                
                    // and remove it from the chunked post list
                    self::remove_chunked_post(self::$query_ids[$chat_id]);
                }

                if( !empty($response) &&
                    ((isset($response->error) && !empty($response->error)) ||
                    (isset($response->statusCode) && ($response->statusCode > 203 || $response->statusCode < 200)))
                    
                ){
                    if(isset($response->error)){
                        if(isset($response->error->message) && !empty($response->error->message)){
                            self::$error_message = esc_html($response->error->message);
                        }

                        if(isset($response->error->type)){
                            if($response->error->type === 'invalid_request_error'){
                                self::$invalid_request = true;
                                self::track_error($response->error->type);
                            }
                        }

                        if(isset($response->error->code)){
                            if($response->error->code === 'rate_limit_exceeded'){
                                self::$rate_limited = true;
                            }elseif($response->error->code === 'invalid_prompt'){
        
                            }elseif($response->error->code === 'insufficient_quota' || $response->error->code === 'billing_hard_limit_reached'){
                                self::$insufficient_quota = true;
                            }elseif($response->error->code === 'invalid_api_key'){
                                self::$invalid_api_key = true;
                            }
                            
                            self::track_error($response->error->code);
                        }
                    }elseif(self::$ai_service_connected){
                        if(isset($response->body)){
                            if(is_string($response->body)){
                                $response->body = json_decode($response->body);
                            }

                            if(isset($response->body->error)){
                                if(isset($response->body->error)){
                                    if($response->body->error === 'User not found'){
                                        self::$user_not_exist = true;
                                    }elseif($response->body->error === 'Insufficient credits'){
                                        self::$insufficient_quota = true;
                                    }elseif($response->body->error === 'Access not valid'){
                                        self::$invalid_api_key = true;
                                    }
                                    
                                    self::track_error($response->body->error);
                                }
                            }
                        }
                    }

                    // format the data for saving
                    $dat_object = (object)array(
                        'response' => array(
                            'status_code' => 200,
                            'body' => $response,
                            'purpose' => self::$purpose
                        ),
                        'custom_id' => self::$query_ids[$chat_id],
                        'id' => 'live_download'
                    );

                    $dat_object = json_encode($dat_object);
                    if(!empty($dat_object)){
                        $dat_object = array($dat_object);
                        self::save_error_log_data('live_download', $dat_object, self::$purpose);
                        // and mark the post as processed if there isn't a temp/quota error
                        if(!self::$insufficient_quota && !self::$rate_limited && !self::$invalid_request && !self::$invalid_api_key && !self::$user_not_exist){
                            self::save_empty_post_data(self::$query_ids[$chat_id], self::$model, self::$purpose);
                        }

                        if(self::$invalid_api_key || self::$user_not_exist){
                            return;
                        }else{
                            continue;
                        }
                    }
                }

                $dat_object = (object)array(
                    'response' => array(
                        'status_code' => 200,
                        'body' => $response
                    ),
                    'custom_id' => self::$query_ids[$chat_id],
                );

                $dat_object = json_encode($dat_object);
                if(!empty($dat_object)){
                    $dat_object = array($dat_object);
                    self::save_site_keywords($dat_object);
//                    self::save_site_post_summaries($dat_object);
                    self::save_site_products($dat_object);
                }

                self::save_response_tokens($dat, self::$purpose);
            }

            // if we have suggestion processing data
            if(!empty(self::$anchor_assessment_ids)){
                // go over it
                foreach(self::$anchor_assessment_ids as $post_sntnce_id => $dat){
                    // and make sure that we've checked off the sentences so we don't get stuck in infinite loops
                    self::log_processed_anchor_sentence($dat['sentence'], self::$origin_post); // if we cut off too soon, and miss some opportunities, we can revisit this...
                }
            }
        }

        // unset any chuncked poasts
        self::$chunked_posts = array();
    }

    /**
     * 
     **/
    public static function live_query_linkwhisper_ai_results($data, $endpoint = ''){
        if(empty($data) || empty(self::$ai_service_connected)){
            return false;
        }

        if($endpoint === 'embeddings'){
            $message_list = array();
            self::$query_ids = array();
            foreach($data as $post_id => $dat){
                if(is_array($dat)){ // we really shouldn't be chunking embedding data...
                    foreach($dat as $chunk_data){
                        $message_list[] = $chunk_data;
                        self::$query_ids[] = $post_id;
                    }
                }else{
                    $message_list[] = $dat;
                    self::$query_ids[] = $post_id;
                }
            }

            $results = self::call_linkwhisper_ai($message_list, '', '', ['dimensions' => Wpil_Settings::get_ai_dimension_limit()]);

            if(!empty($results)){
                foreach($results as $key => $dat){
                    
                    $response = self::decode($dat);
                    if(!empty($response) &&
                        ((isset($response->error) && !empty($response->error)) ||
                        (isset($response->statusCode) && ($response->statusCode > 203 || $response->statusCode < 200)))
                    ){
                        if(isset($response->error)){
                            if(isset($response->error->message) && !empty($response->error->message)){
                                self::$error_message = esc_html($response->error->message);
                            }

                            if(isset($response->error->type)){
                                if($response->error->type === 'invalid_request_error'){
                                    self::$invalid_request = true;
                                    self::track_error($response->error->type);
                                }
                            }

                            if(isset($response->error->code)){
                                if($response->error->code === 'rate_limit_exceeded'){
                                    self::$rate_limited = true;
                                }elseif($response->error->code === 'invalid_prompt'){
            
                                }elseif($response->error->code === 'insufficient_quota' || $response->error->code === 'billing_hard_limit_reached'){
                                    self::$insufficient_quota = true;
                                }elseif($response->error->code === 'invalid_api_key'){
                                    self::$invalid_api_key = true;
                                }
                                
                                self::track_error($response->error->code);
                            }
                        }elseif(self::$ai_service_connected){
                            if(isset($response->body)){
                                if(is_string($response->body)){
                                    $response->body = json_decode($response->body);
                                }

                                if(isset($response->body->error)){
                                    if(isset($response->body->error)){
                                        if($response->body->error === 'User not found'){
                                            self::$user_not_exist = true;
                                        }elseif($response->body->error === 'Insufficient credits'){
                                            self::$insufficient_quota = true;
                                        }elseif($response->body->error === 'Access not valid'){
                                            self::$invalid_api_key = true;
                                        }
                                        
                                        self::track_error($response->body->error);
                                    }
                                }
                            }
                        }

                        // format the data for saving
                        $dat_object = (object)array(
                            'response' => array(
                                'status_code' => 200,
                                'body' => $response,
                                'purpose' => self::$purpose
                            ),
                            'custom_id' => self::$query_ids[$key],
                            'id' => 'live_download'
                        );
    
                        $dat_object = json_encode($dat_object);
                        if(!empty($dat_object)){
                            $dat_object = array($dat_object);
                            self::save_error_log_data('live_download', $dat_object, self::$purpose);
                            // and mark the post as processed if there isn't a temp/quota error
                            if(!self::$insufficient_quota && !self::$rate_limited && !self::$invalid_request && !self::$invalid_api_key && !self::$user_not_exist){
                                self::save_empty_post_data(self::$query_ids[$key], self::$model, self::$purpose);
                            }

                            if(self::$invalid_api_key || self::$user_not_exist){
                                return;
                            }else{
                                continue;
                            }
                        }
                    }
        
                    $dat_object = (object)array(
                        'response' => array(
                            'status_code' => 200,
                            'body' => $response
                        ),
                        'custom_id' => self::$query_ids[$key],
                    );
        
                    $dat_object = json_encode($dat_object);
                    
                    if(!empty($dat_object)){
                        $dat_object = array($dat_object);
                        self::save_site_embeddings($dat_object);
                    }
        
                    self::save_response_tokens($dat, self::$purpose, false, isset(self::$query_ids[$key]) ? self::$query_ids[$key] : '');
                }
            }
        }else{
            $message_list = array();
            self::$query_ids = array();
            foreach($data as $post_id => $dat){
                if(is_array($dat)){
                    foreach($dat as $chunk_data){
                        $message_list[] = $chunk_data;
                        self::$query_ids[] = $post_id;
                        self::$chunked_posts[$post_id] = true;
                    }
                }else{
                    $message_list[] = $dat;
                    self::$query_ids[] = $post_id;
                }
            }

            $args = array(
                'model' => self::$model,
                'response_format' => self::create_response_format('live_download', self::create_response_indexes(self::$purpose), self::$model, (self::$purpose === 'assess-sentence-anchors')),
                'message_list' => $message_list,
                'temperature' => 0,
                //'top_p' => 0.1,
                'frequency_penalty' => 0,
                'presence_penalty' => -2.0
            );

            $chat = self::call_linkwhisper_ai($message_list);
            $merge_data = array();
            foreach($chat as $chat_id => $dat){
                // if there was no response but we did have content to supply
                if(empty($dat) && isset($args['message_list'][$chat_id]) && !empty($args['message_list'][$chat_id])){
                    // if we've already chunked the post
                    if(isset(self::$chunked_posts[self::$query_ids[$chat_id]])){
                        // remove it from the chunked post list
                        self::remove_chunked_post(self::$query_ids[$chat_id]);
                        // save the empty data
                        self::save_empty_post_data(self::$query_ids[$chat_id], self::$model, self::$purpose);
                        // and continue
                        continue;
                    }

                    // assume that we need to chunck the request to get it past OAI
                    self::save_chunked_post(self::$query_ids[$chat_id]);
                    continue;
                }

                $response = self::decode($dat);

                if(!empty($response) && isset(self::$chunked_posts[self::$query_ids[$chat_id]])){
                    $next = $chat_id + 1;
                    $sub_dat = self::decode($response->choices[0]->message->content);

                    // add the response content to the chunk merge data
                    if(!isset($merge_data[self::$query_ids[$chat_id]])){
                        $merge_data[self::$query_ids[$chat_id]] = array(
                            'keywords' => array(),
                            'keyword-count' => 0,
                            'products' => array(),
                            'product-count' => 0
                        );
                    }

                    if(!empty($sub_dat) && isset($sub_dat->results)){
                        if(isset($sub_dat->results->keywords)){ 
                            $merge_data[self::$query_ids[$chat_id]]['keywords'] = array_unique(array_merge($merge_data[self::$query_ids[$chat_id]]['keywords'], explode(',', $sub_dat->results->keywords)));
                            $merge_data[self::$query_ids[$chat_id]]['keyword-count'] = count($merge_data[self::$query_ids[$chat_id]]['keywords']);
                        }
                        if(isset($sub_dat->results->products)){ 
                            $merge_data[self::$query_ids[$chat_id]]['products'] = array_unique(array_merge($merge_data[self::$query_ids[$chat_id]]['products'], explode(',', $sub_dat->results->products)));
                            $merge_data[self::$query_ids[$chat_id]]['product-count'] = count($merge_data[self::$query_ids[$chat_id]]['products']);
                        }
                    }

                    if(
                        isset($chat[$next]) && // if there's another item in the chat
                        isset(self::$query_ids[$next]) && // and we have an id for it
                        isset(self::$chunked_posts[self::$query_ids[$next]]) && // and the next item is chunked
                        self::$query_ids[$chat_id] === self::$query_ids[$next] // and the id is the same as this one
                    ){
                        // note the tokens used for this request
                        self::save_response_tokens($dat, self::$purpose);
                        // and continue on to the next item
                        continue;
                    }

                    // if this is the last item, update the response content
                    $merge_data[self::$query_ids[$chat_id]]['keywords'] = implode(',', $merge_data[self::$query_ids[$chat_id]]['keywords']);
                    $merge_data[self::$query_ids[$chat_id]]['products'] = implode(',', $merge_data[self::$query_ids[$chat_id]]['products']);
                    $response->choices[0]->message->content = json_encode(array('results' => $merge_data[self::$query_ids[$chat_id]]));
                
                    // and remove it from the chunked post list
                    self::remove_chunked_post(self::$query_ids[$chat_id]);
                }

                if(!empty($response) &&
                    ((isset($response->error) && !empty($response->error)) ||
                    (isset($response->statusCode) && ($response->statusCode > 203 || $response->statusCode < 200)))
                ){
                    if(isset($response->error)){
                        if(isset($response->error->message) && !empty($response->error->message)){
                            self::$error_message = esc_html($response->error->message);
                        }

                        if(isset($response->error->type)){
                            if($response->error->type === 'invalid_request_error'){
                                self::$invalid_request = true;
                                self::track_error($response->error->type);
                            }
                        }

                        if(isset($response->error->code)){
                            if($response->error->code === 'rate_limit_exceeded'){
                                self::$rate_limited = true;
                            }elseif($response->error->code === 'invalid_prompt'){
        
                            }elseif($response->error->code === 'insufficient_quota' || $response->error->code === 'billing_hard_limit_reached'){
                                self::$insufficient_quota = true;
                            }elseif($response->error->code === 'invalid_api_key'){
                                self::$invalid_api_key = true;
                            }
                            
                            self::track_error($response->error->code);
                        }
                    }elseif(self::$ai_service_connected){
                        if(isset($response->body)){
                            if(is_string($response->body)){
                                $response->body = json_decode($response->body);
                            }

                            if(isset($response->body->error)){
                                if(isset($response->body->error)){
                                    if($response->body->error === 'User not found'){
                                        self::$user_not_exist = true;
                                    }elseif($response->body->error === 'Insufficient credits'){
                                        self::$insufficient_quota = true;
                                    }elseif($response->body->error === 'Access not valid'){
                                        self::$invalid_api_key = true;
                                    }
                                    
                                    self::track_error($response->body->error);
                                }
                            }
                        }
                    }

                    // format the data for saving
                    $dat_object = (object)array(
                        'response' => array(
                            'status_code' => 200,
                            'body' => $response,
                            'purpose' => self::$purpose
                        ),
                        'custom_id' => self::$query_ids[$chat_id],
                        'id' => 'live_download'
                    );

                    $dat_object = json_encode($dat_object);
                    if(!empty($dat_object)){
                        $dat_object = array($dat_object);
                        self::save_error_log_data('live_download', $dat_object, self::$purpose);
                        // and mark the post as processed if there isn't a temp/quota error
                        if(!self::$insufficient_quota && !self::$rate_limited && !self::$invalid_request && !self::$invalid_api_key && !self::$user_not_exist){
                            self::save_empty_post_data(self::$query_ids[$chat_id], self::$model, self::$purpose);
                        }

                        if(self::$invalid_api_key || self::$user_not_exist){
                            return;
                        }else{
                            continue;
                        }
                    }
                }

                $dat_object = (object)array(
                    'response' => array(
                        'status_code' => 200,
                        'body' => $response
                    ),
                    'custom_id' => self::$query_ids[$chat_id],
                );

                $dat_object = json_encode($dat_object);
                if(!empty($dat_object)){
                    $dat_object = array($dat_object);
                    self::save_site_keywords($dat_object);
//                    self::save_site_post_summaries($dat_object);
                    self::save_site_products($dat_object);
                }

                self::save_response_tokens($dat, self::$purpose);
            }

            // if we have suggestion processing data
            if(!empty(self::$anchor_assessment_ids)){
                // go over it
                foreach(self::$anchor_assessment_ids as $post_sntnce_id => $dat){
                    // and make sure that we've checked off the sentences so we don't get stuck in infinite loops
                    self::log_processed_anchor_sentence($dat['sentence'], self::$origin_post); // if we cut off too soon, and miss some opportunities, we can revisit this...
                }
            }
        }

        // unset any chuncked poasts
        self::$chunked_posts = array();
    }

    /**
     * Gets the embeddings for a specific post, broken down by phrase
     * @param Wpil_Post $post
     **/
    public static function live_query_single_post_embedding_data($post = null){
        if(empty($post) || !is_a($post, 'Wpil_Model_Post') || (empty(self::$ai) && !self::$ai_service_connected)){
            return false;
        }

        $model = Wpil_Settings::getChatGPTVersion('create-post-embeddings');
        $dimensions = Wpil_Settings::get_ai_dimension_limit();
        self::$purpose = 'create-post-sentence-embeddings'; // TODO: create link cleanup that will check to see if the links have changed between the processed and new text and will tell the stupid AI if it should reprocessed the posit content.
        $phrases = Wpil_Suggestion::getPhrases($post->getContent(), false, array(), false, array(), ('sentence_text' === Wpil_Suggestion::get_phrase_text_prop()));
        //$phrases = Wpil_Suggestion::getPhrases($post->getContent()); // TODO: Review and see if we need to id text instead of sentence_text
        $message_list = array();

        $phrase_list = array();
        foreach($phrases as $phrase){
            $text = Wpil_Suggestion::get_ai_phrase_text($phrase);
            if(empty($text) || strlen($text) < 3 || Wpil_Word::getWordCount($text) < 4){ // only process sentences that are long enough to matter
                continue;
            }
            
            $phrase_list[$text] = true;
        }

        if(empty($phrase_list)){
            self::save_empty_phrase_embedding($post->get_pid(), $model);
            return array();
        }

        if(self::$ai_service_connected){
            foreach($phrase_list as $text => $dat){
                $message_list[] = $text;
                self::$query_ids[] = $post->get_pid(); // get the pid for this post because any errors will be indexed for it
            }

            $results = self::call_linkwhisper_ai($message_list, $model, '', ['dimensions' => $dimensions]);
        }else{
            foreach($phrase_list as $text => $dat){
                $message_list[] = array(
                    "model" => $model,
                    "input" => $text,
                    "dimensions" => $dimensions
                );

                self::$query_ids[] = $post->get_pid(); // get the pid for this post because any errors will be indexed for it
            }

            $args = array(
                'message_list' => $message_list,
            );

            $results = self::$ai->embeddings($args, true);
        }

        $embedding_data = array();
        if(!empty($results)){
            $inds = array_keys($phrase_list);
            foreach($results as $key => $dat){
                $response = self::decode($dat);
                if( !empty($response) &&
                    ((isset($response->error) && !empty($response->error)) ||
                    (isset($response->statusCode) && ($response->statusCode > 203 || $response->statusCode < 200)))
                ){
                    if(isset($response->error)){
                        if(isset($response->error->message) && !empty($response->error->message)){
                            self::$error_message = esc_html($response->error->message);
                        }

                        if(isset($response->error->type)){
                            if($response->error->type === 'invalid_request_error'){
                                self::$invalid_request = true;
                                self::track_error($response->error->type);
                            }
                        }

                        if(isset($response->error->code)){
                            if($response->error->code === 'rate_limit_exceeded'){
                                self::$rate_limited = true;
                            }elseif($response->error->code === 'invalid_prompt'){

                            }elseif($response->error->code === 'insufficient_quota' || $response->error->code === 'billing_hard_limit_reached'){
                                self::$insufficient_quota = true;
                            }elseif($response->error->code === 'invalid_api_key'){
                                self::$invalid_api_key = true;
                            }
                            
                            self::track_error($response->error->code);
                        }
                    }elseif(self::$ai_service_connected){
                        if(isset($response->body)){
                            if(is_string($response->body)){
                                $response->body = json_decode($response->body);
                            }

                            if(isset($response->body->error)){
                                if(isset($response->body->error)){
                                    if($response->body->error === 'User not found'){
                                        self::$user_not_exist = true;
                                    }elseif($response->body->error === 'Insufficient credits'){
                                        self::$insufficient_quota = true;
                                    }elseif($response->body->error === 'Access not valid'){
                                        self::$invalid_api_key = true;
                                    }
                                    
                                    self::track_error($response->body->error);
                                }
                            }
                        }
                    }

                    // format the data for saving
                    $dat_object = (object)array(
                        'response' => array(
                            'status_code' => 200,
                            'body' => $response,
                            'purpose' => self::$purpose
                        ),
                        'custom_id' => isset(self::$query_ids[$key]) ? self::$query_ids[$key]: '', // TODO: Keep an eye on this and make sure that it doesn't allow endless loops
                        'id' => 'live_download'
                    );

                    $dat_object = json_encode($dat_object);
                    if(!empty($dat_object)){
                        $dat_object = array($dat_object);
                        self::save_error_log_data('live_download', $dat_object, self::$purpose);
                        // and mark the post as processed if there isn't a temp/quota error
                        if(!self::$insufficient_quota && !self::$rate_limited && !self::$invalid_request && !self::$invalid_api_key || self::$user_not_exist){
                            //self::save_empty_post_data(self::$query_ids[$key], self::$model, self::$purpose);
                        }

                        if(self::$invalid_api_key || self::$user_not_exist){
                            return;
                        }else{
                            continue;
                        }
                    }
                }

                if(isset($response->data) && !empty($response->data)){
                    $phrase_id = $inds[$key];
                    $embedding_data[$phrase_id] = $response->data;
                }
                self::save_response_tokens($dat, self::$purpose, false, isset(self::$query_ids[$key]) ? self::$query_ids[$key] : '');
            }
        }

        return $embedding_data;
    }

    /**
     * 
     **/
    public static function process_streamed_data($handle_id, $content, $info){
        $skip_streaming = array(
            'create-post-sentence-embeddings'
        );

        // if the current action isn't supposed to be streamed
        if(!empty(self::$purpose) && in_array(self::$purpose, $skip_streaming)){
            // return now
            return false;
        }

        // for the time being, don't process chunked posts
        if(!empty(self::$chunked_posts) && isset(self::$chunked_posts[$handle_id])){
            return false;
        }

        if(empty($content) || empty($info) || empty($info['url'])){
            return false;
        }

        // if the response wasn't successful
        if($info['http_code'] == 429 || $info['http_code'] > 499){
            return true; // return 'true' to mark the post as processed here so it doesn't get processed by the code that checks off failed posts
        }

        if(false !== strpos($info['url'], 'embeddings') || (false !== strpos($info['url'], 'api.linkwhisper.com') && self::$purpose === 'create-post-embeddings')){
            $response = self::decode($content);
            if( !empty($response) && 
                ((isset($response->error) && !empty($response->error)) ||
                (isset($response->statusCode) && ($response->statusCode > 203 || $response->statusCode < 200)))
            ){
                if(isset($response->error)){
                    if(isset($response->error->message) && !empty($response->error->message)){
                        self::$error_message = esc_html($response->error->message);
                    }
                    if(isset($response->error->type)){
                        if($response->error->type === 'invalid_request_error'){
                            self::$invalid_request = true;
                            self::track_error($response->error->type);
                        }
                    }

                    if(isset($response->error->code)){
                        if($response->error->code === 'rate_limit_exceeded'){
                            self::$rate_limited = true;
                        }elseif($response->error->code === 'invalid_prompt'){

                        }elseif($response->error->code === 'insufficient_quota' || $response->error->code === 'billing_hard_limit_reached'){
                            self::$insufficient_quota = true;
                        }elseif($response->error->code === 'invalid_api_key'){
                            self::$invalid_api_key = true;
                        }
                        
                        self::track_error($response->error->code);
                    }
                }elseif(self::$ai_service_connected){
                    if(isset($response->body)){
                        if(is_string($response->body)){
                            $response->body = json_decode($response->body);
                        }

                        if(isset($response->body->error)){
                            if(isset($response->body->error)){
                                if($response->body->error === 'User not found'){
                                    self::$user_not_exist = true;
                                }elseif($response->body->error === 'Insufficient credits'){
                                    self::$insufficient_quota = true;
                                }elseif($response->body->error === 'Access not valid'){
                                    self::$invalid_api_key = true;
                                }
                                
                                self::track_error($response->body->error);
                            }
                        }
                    }
                }

                // format the data for saving
                $dat_object = (object)array(
                    'response' => array(
                        'status_code' => 200,
                        'body' => $response,
                        'purpose' => self::$purpose
                    ),
                    'custom_id' => self::$query_ids[$handle_id],
                    'id' => 'live_download'
                );

                $dat_object = json_encode($dat_object);
                if(!empty($dat_object)){
                    $dat_object = array($dat_object);
                    self::save_error_log_data('live_download', $dat_object, self::$purpose);
                    // and mark the post as processed if there isn't a temp/quota error
                    if( !self::$insufficient_quota && 
                        !self::$rate_limited && 
                        !self::$invalid_request && 
                        !self::$invalid_api_key && 
                        !self::$user_not_exist || 
                        (self::$ai_service_connected) // or if we're using the LW AI service 
                    ){
                        self::save_empty_post_data(self::$query_ids[$handle_id], self::$model, self::$purpose);
                    }
                    return true;
                }
            }

            $dat_object = (object)array(
                'response' => array(
                    'status_code' => 200,
                    'body' => $response
                ),
                'custom_id' => self::$query_ids[$handle_id],
            );

            $dat_object = json_encode($dat_object);
            if(!empty($dat_object)){
                $dat_object = array($dat_object);
                self::save_site_embeddings($dat_object);
            }

            self::save_response_tokens($content, self::$purpose, false, isset(self::$query_ids[$handle_id]) ? self::$query_ids[$handle_id] : '');
        }else{
            // if there was no response but we did have content to supply
            if(empty($content) && isset(self::$query_ids[$handle_id]) && !empty(self::$query_ids[$handle_id])){
                // assume that we need to chunck the request to get it past OAI
                self::save_chunked_post(self::$query_ids[$handle_id]);
                return true;
            }

            $response = self::decode($content);
            if(!empty($response) && 
                ((isset($response->error) && !empty($response->error)) ||
                (isset($response->statusCode) && ($response->statusCode > 203 || $response->statusCode < 200)))
            ){
                if(isset($response->error)){
                    if(isset($response->error->message) && !empty($response->error->message)){
                        self::$error_message = esc_html($response->error->message);
                    }

                    if(isset($response->error->type)){
                        if($response->error->type === 'invalid_request_error'){
                            self::$invalid_request = true;
                            self::track_error($response->error->type);
                        }
                    }

                    if(isset($response->error->code)){
                        if($response->error->code === 'rate_limit_exceeded'){
                            self::$rate_limited = true;
                        }elseif($response->error->code === 'invalid_prompt'){

                        }elseif($response->error->code === 'insufficient_quota' || $response->error->code === 'billing_hard_limit_reached'){
                            self::$insufficient_quota = true;
                        }elseif($response->error->code === 'invalid_api_key'){
                            self::$invalid_api_key = true;
                        }
                        
                        self::track_error($response->error->code);
                    }
                }elseif(self::$ai_service_connected){
                    if(isset($response->body)){
                        if(is_string($response->body)){
                            $response->body = json_decode($response->body);
                        }

                        if(isset($response->body->error)){
                            if(isset($response->body->error)){
                                if($response->body->error === 'User not found'){
                                    self::$user_not_exist = true;
                                }elseif($response->body->error === 'Insufficient credits'){
                                    self::$insufficient_quota = true;
                                }elseif($response->body->error === 'Access not valid'){
                                    self::$invalid_api_key = true;
                                }
                                
                                self::track_error($response->body->error);
                            }
                        }
                    }
                }

                // format the data for saving
                $dat_object = (object)array(
                    'response' => array(
                        'status_code' => 200,
                        'body' => $response,
                        'purpose' => self::$purpose
                    ),
                    'custom_id' => self::$query_ids[$handle_id],
                    'id' => 'live_download'
                );

                $dat_object = json_encode($dat_object);
                if(!empty($dat_object)){
                    $dat_object = array($dat_object);
                    self::save_error_log_data('live_download', $dat_object, self::$purpose);
                    // and mark the post as processed if there isn't a temp/quota error
                    if(!self::$insufficient_quota && !self::$rate_limited && !self::$invalid_request && !self::$invalid_api_key && !self::$user_not_exist){
                        self::save_empty_post_data(self::$query_ids[$handle_id], self::$model, self::$purpose);
                    }
                    return true;
                }
            }

            $dat_object = (object)array(
                'response' => array(
                    'status_code' => 200,
                    'body' => $response
                ),
                'custom_id' => self::$query_ids[$handle_id],
            );

            $dat_object = json_encode($dat_object);
            if(!empty($dat_object)){
                $dat_object = array($dat_object);

                // TODO: make more elegant in the future
                if(self::$purpose === 'assess-sentence-anchors'){
                    self::save_ai_suggestion_words($dat_object);
                }elseif(self::$purpose === 'assess-inbound-links'){
                    self::save_ai_linking_suggestions($dat_object);
                }elseif(self::$purpose === 'assess-outbound-links'){
                    //self::save_ai_linking_suggestions($dat_object); // TODO: configure if we move to mlulltiple process runs
                }else{
                    self::save_site_keywords($dat_object);
                    //self::save_site_post_summaries($dat_object);
                    self::save_site_products($dat_object);
                }
            }

            self::save_response_tokens($content, self::$purpose);
        }

        return true;
    }

    /**
     * Calculates the post embeddings for the posts on the site.
     * Handles all the process running required to make the calculations.
     **/
    public static function calculate_post_embeddings(){
        $processed_embeddings = array();

        $large_site = self::get_total_processable_posts() > 4000;
        if($large_site){
            $embedding_data = self::get_post_embedding_data(); // currently pulling all data, will batch in the future
        }else{
            $embedding_data = self::get_post_embedding_data(true); // currently pulling all data, will batch in the future
        }

        $stored_posts = self::get_calculated_embedding_post_ids();

        if(empty($embedding_data)){
            return false;
        }

        $count = 0;
        foreach($embedding_data as $key => $dat){
            if(Wpil_Base::overTimeLimit(15) || Wpil_Toolbox::is_over_memory_limit()){
                break;
            }

            $id = $dat->post_type . '_' . $dat->post_id;

            // if the post is already stored in the embedding table
            if(isset($stored_posts[$id])){
                continue;
            }

            if(!isset($processed_embeddings[$id])){
                $processed_embeddings[$id] = array();
            }

            if($large_site){
                $dat->embed_data = Wpil_Toolbox::json_decompress($dat->embed_data, null, true);
            }
            foreach($embedding_data as $d){
                $sub_id = $d->post_type . '_' . $d->post_id;

                // if the sub item is the main item
                if($id === $sub_id){
                    // skip to the next because we don't need to determine how related the post is to itself
                    continue;
                }

                if(!isset($processed_embeddings[$id]['embeddings'])){
                    $processed_embeddings[$id]['embeddings'] = array();
                    $processed_embeddings[$id]['model_version'] = $dat->model_version;
                }

                if($large_site){
                    $d->embed_data = Wpil_Toolbox::json_decompress($d->embed_data, null, true);
                    gc_collect_cycles();
                }
                $processed_embeddings[$id]['embeddings'][$sub_id] = self::compare_post_embeddings($dat, $d);
            }

            if($count > 100){
                self::save_calculated_embedding_data($processed_embeddings);
                $count = 0;
                $processed_embeddings = [];
            }elseif(Wpil_Toolbox::is_over_memory_limit() && !empty($processed_embeddings)){
                self::save_calculated_embedding_data($processed_embeddings);
                $count++;
                break;
            }

            $count++;
        }

        if(!empty($processed_embeddings)){
            self::save_calculated_embedding_data($processed_embeddings);
        }

        return $count;
    }

    /**
     * Calculates the post embeddings for the posts on the site.
     * Handles all the process running required to make the calculations.
     **/
    public static function stepped_calculate_post_embeddings(){
        $processed_embeddings = array();
        // get the latest index
        $last_embedding_index = self::get_last_embedding_index();
        $batch_limit = Wpil_Settings::get_ai_process_limit('create-post-embeddings', true);
        $lowest_ind = 0;
        $saving = array();

        // get a batch of posts that are less than the last index
        $calc_process_posts = self::get_offset_embedding_calc_posts($last_embedding_index, $batch_limit, true);

        // find the lowest index info
        if(!empty($calc_process_posts)){
            foreach($calc_process_posts as $dat){
                $id = $dat->post_type . '_' . $dat->post_id;
                $processed_embeddings[$id] = $dat;
            }
            unset($calc_process_posts);
        }else{
            return false;
        }

        while(!Wpil_Base::overTimeLimit(15)){
            // find the lowest index info
            $lowest_ind = $last_embedding_index;
            foreach($processed_embeddings as $dat){
                if(is_null($dat->calc_index)){
                    $dat->calc_index = 0;
                }

                if(isset($dat->calc_index) && $dat->calc_index < $lowest_ind){
                    $lowest_ind = $dat->calc_index;
                }
            }

            // pull a batch of embedding data that picks up after the last
            $batch_embeddings = self::get_post_embedding_data(true, $lowest_ind, $batch_limit);

            if(empty($batch_embeddings)){
                break;
            }

            $count = 0;
            $saving = array();
            foreach($processed_embeddings as $key => $dat){
                if(Wpil_Base::overTimeLimit(15)){
                    break;
                }

                $id = $dat->post_type . '_' . $dat->post_id;

                if(!isset($processed_embeddings[$id])){
                    $processed_embeddings[$id] = array();
                }

                foreach($batch_embeddings as $d){
                    $sub_id = $d->post_type . '_' . $d->post_id;

                    // if this item already has this embedding data calculated
                    if($processed_embeddings[$id]->calc_index >= $d->embed_index){
                        // skip to the next to save time
                        continue;
                    }

                    // if the sub item is the main item
                    if($id === $sub_id){
                        // tag the embed index so we know it's counted
                        if($processed_embeddings[$id]->calc_index < $d->embed_index){
                            $processed_embeddings[$id]->calc_index = $d->embed_index;
                        }
                        // skip to the next because we don't need to determine how related the post is to itself
                        continue;
                    }

                    if(!isset($processed_embeddings[$id]->calculation) || empty($processed_embeddings[$id]->calculation)){
                        $processed_embeddings[$id]->calculation = array();
                    }

                    $calculation = self::compare_post_embeddings($dat, $d);

                    // if we pass the threshold for minimum relatability
                    if($calculation > 0.40){ // TODO: make into a setting if we have trouble with generating enough suggestions
                        // add it to the list
                        $processed_embeddings[$id]->calculation[$sub_id] = $calculation;
                    }
                    
                    if($processed_embeddings[$id]->calc_index < $d->embed_index){
                        $processed_embeddings[$id]->calc_index = $d->embed_index;
                    }

                    $processed_embeddings[$id]->calc_count += 1;
                }
                $saving[$id] = $processed_embeddings[$id];
                
                // if we're at the memory breakpoint
                if(Wpil_Toolbox::is_over_memory_limit()){
                    // save the data to clear it
                    self::save_calculated_embedding_data($saving, true);
                    $count = 0;
                    $saving = [];
                }

                $count++;
            }

            if(!empty($saving)){
                self::save_calculated_embedding_data($saving, true);
                $saving = [];
            }
        }

        // if we _still_ have data to save
        if(!empty($saving)){
            // save it here
            self::save_calculated_embedding_data($saving, true);
        }

        return true;
    }

    /**
     * 
     **/
    public static function get_last_embedding_index(){
        global $wpdb;
        $table = $wpdb->prefix . 'wpil_ai_embedding_data';

        $index = self::get_last_embedding_id_lock();

        if(!empty($index)){
            return $index;
        }

        $index = $wpdb->get_var("SELECT `embed_index` FROM {$table} ORDER BY `embed_index` DESC LIMIT 1");

        self::set_last_embedding_id_lock($index);

        return (!empty($index)) ? (int)$index: 0;
    }

    private static function get_last_embedding_id_lock(){
        $lock = get_transient('wpil_last_embedding_index_lock');
        return (!empty($lock)) ? (int)$lock: 0;
    }

    private static function set_last_embedding_id_lock($id = null){
        if(empty($id)){
            delete_transient('wpil_last_embedding_index_lock');
        }else{
            set_transient('wpil_last_embedding_index_lock', (int)$id, DAY_IN_SECONDS);
        }
    }

    /**
     * 
     **/
    public static function get_offset_embedding_calc_posts($embedding_index = 0, $limit = 0, $decode = false){
        global $wpdb;
        $embed_table = $wpdb->prefix . 'wpil_ai_embedding_data';
        $calc_table = $wpdb->prefix . 'wpil_ai_embedding_calculation_data';

        $limit = (int)$limit;
        $embedding_index = (int)$embedding_index;

        if(empty($limit)){
            $limit = 1000;
        }

        $data = $wpdb->get_results("SELECT a.post_id, a.post_type, a.data_type, a.embed_data, a.model_version, b.calculation, b.calc_index, b.calc_count  FROM 
            {$embed_table} a LEFT JOIN {$calc_table} b ON a.post_id = b.post_id AND a.post_type = b.post_type 
            WHERE b.calc_index < {$embedding_index} OR ISNULL(b.calc_index) LIMIT {$limit}");

        if($decode && !empty($data)){
            foreach($data as $key => $dat){
                if(isset($dat->calculation)){
                    $data[$key]->calculation = Wpil_Toolbox::json_decompress($dat->calculation, true);
                }

                if(isset($dat->embed_data)){
                    $data[$key]->embed_data = Wpil_Toolbox::json_decompress($dat->embed_data, true, true);
                }
            }
        }

        return $data;
    }
    
    /**
     * Gets the raw embedding data for psots that we're currently calculating the AI relationship scor fore
     **/
    public static function get_target_embedding_posts($post_ids = array()){
        global $wpdb;
        $table = $wpdb->prefix . 'wpil_ai_embedding_data';
        $target_posts = array();

        if(empty($post_ids)){
            return $target_posts;
        }

        $query = "";
        foreach($post_ids as $type => $post_ids){
            $ids = array_filter(array_map(function($id){ return (int)$id; }, $post_ids));

            if(!empty($ids) && ($type === 'post' || $type === 'term')){
                $ids = implode(',', $ids);
                $query .= !empty($query) ? " OR ": "";
                $query .= "(`post_type` = {$type} AND `post_id` IN ({$ids})) ";
            }
        }

        if(!empty($query)){
            $target_posts = $wpdb->get_results("SELECT * FROM {$table} WHERE {$query}");
        }

        return $target_posts;
    }

    /**
     * @param object $post1
     * @param object $post2
     **/
    public static function compare_post_embeddings($post1 = array(), $post2 = array(), $forbid_cache = false){
        $dimension1_count = count($post1->embed_data);
        $dimension2_count = count($post2->embed_data);

        if($dimension1_count > $dimension2_count){
            $post1->embed_data = self::reduce_embedding_dimensions($post1->embed_data, $dimension2_count);
        }elseif($dimension2_count > $dimension1_count){
            $post2->embed_data = self::reduce_embedding_dimensions($post2->embed_data, $dimension1_count);
        }

        $dot_product = self::get_cached_dot_product($post1, $post2, $forbid_cache);
        $magnitude_a = self::get_cached_magnatude($post1, $forbid_cache);
        $magnitude_b = self::get_cached_magnatude($post2, $forbid_cache);
        $similarity = $dot_product / ($magnitude_a * $magnitude_b);
        return number_format($similarity, 12, '.', '');
    }

    /**
     * @param object $post1
     * @param object $post2
     **/
    public static function get_cached_dot_product($post1 = array(), $post2 = array(), $forbid_cache = false){
        if(isset($post1->sentence)){
            $id = md5($post1->sentence) . '_' . $post2->post_id . '_' . $post2->post_type;
        }elseif(isset($post2->sentence)){
            $id = md5($post2->sentence) . '_' . $post1->post_id . '_' . $post1->post_type;
        }else{
            if($post1->post_id > $post2->post_id){
                $id = ($post1->post_id . '_' . $post1->post_type) . '_' . ($post2->post_id . '_' . $post2->post_type);
            }else{
                $id = ($post2->post_id . '_' . $post2->post_type) . '_' . ($post1->post_id . '_' . $post1->post_type);
            }
        }

        // if we don't have the magnatude cached
        if(!isset(self::$dot_product_cache[$id])){
            $dot_product = 0;
            foreach($post1->embed_data as $key => $p1s){
                if(isset($post2->embed_data[$key])){
                    $dot_product += $p1s * $post2->embed_data[$key];
                }
            }

            // if we're not caching it
            if($forbid_cache){
                // return it here
                return $dot_product;
            }

            // create and cache it
            self::$dot_product_cache[$id] = $dot_product;
        }

        return self::$dot_product_cache[$id];
    }

    /**
     * @param object $embedded_post
     **/
    public static function get_cached_magnatude($embedded_post = array(), $forbid_cache = false){
        $id = (isset($embedded_post->sentence) && !empty($embedded_post->sentence)) ? md5($embedded_post->sentence): $embedded_post->post_type . '_' . $embedded_post->post_id;

        // if we don't have the magnatude cached
        if(!isset(self::$magnatude_cache[$id])){
            // create it
            $magnatude = sqrt(array_sum(array_map(function($x){return $x * $x;}, Wpil_Toolbox::json_decompress($embedded_post->embed_data, null, true))));

            // if we're not supposed to cache it
            if($forbid_cache){
                // return it now
                return $magnatude;
            }

            // otherwise, cache it for future use
            self::$magnatude_cache[$id] = $magnatude;
        }

        return self::$magnatude_cache[$id];
    }


    /**
     * TODO: redescribe
     * Calculates the post embeddings for the posts on the site.
     * Handles all the process running required to make the calculations.
     * 
     * TODO: Make able to take post language status into account
     * 
     * @param Wpil_Model_Post $target_post The post whose sentences we're calculating relations for
     **/
    public static function stepped_calculate_phrase_embeddings($target_post = array(), $return_calculations = false){
        if(empty($target_post) || !is_a($target_post, 'Wpil_Model_Post')){
            return ($return_calculations) ? 0: false;
        }

        // get the latest index
        $last_embedding_index = self::get_last_embedding_index();
        $batch_limit = Wpil_Settings::get_ai_process_limit('create-post-embeddings', true);
        $lowest_ind = 0;
        $completed = false;

        // get a batch of posts that are less than the last index
        $phrase_embeddings = self::get_single_post_embedding_data($target_post, true);
        $calculated_phrase_data = self::get_embedding_calc_phrases($target_post, true, true);

        if(empty($phrase_embeddings)){
            return ($return_calculations) ? 0: false;
        }

        if(empty($calculated_phrase_data)){
            $first_sentence = reset($phrase_embeddings->embed_data);
            $calculated_phrase_data = (object) array(
                'post_id' => $target_post->id,
                'post_type' => $target_post->type,
                'data_type' => ($target_post->type === 'post' ? 1: 0),
                'calculation' => array(),
                'calc_index' => array(),
                'calc_count' => 0,
                'process_time' => time(),
                'model_version' => $phrase_embeddings->model_version,
                'dimension_count' => count($first_sentence)
            );
        }

        // if we don't have calculation data
        if(empty($calculated_phrase_data->calculation)){
            // setup the calculation indexes
            foreach($phrase_embeddings->embed_data as $sentence => $dat){
                $calculated_phrase_data->calculation[$sentence] = array();
                $calculated_phrase_data->calc_index[$sentence] = 0;
            }
        }

        $id = $phrase_embeddings->post_type . '_' . $phrase_embeddings->post_id;
        while(!Wpil_Base::overTimeLimit(15)){

            // find the lowest index info
            $lowest_ind = $last_embedding_index;
            foreach($calculated_phrase_data->calc_index as $snt => $dat){
                if(is_null($dat)){
                    $dat = 0;
                }

                if($dat < $lowest_ind){
                    $lowest_ind = $dat;
                }
            }

            // pull a batch of embedding data that picks up after the last
            $batch_embeddings = self::get_post_embedding_data(true, $lowest_ind, $batch_limit);


            if(empty($batch_embeddings)){
                $completed = true;
                break;
            }

            $count = 0;
            foreach($phrase_embeddings->embed_data as $sentence => $dat){
                if(Wpil_Base::overTimeLimit(15)){
                    break;
                }

                $phrase_object = (object) array(
                    'sentence' => $sentence,
                    'embed_data' => $dat
                );

                foreach($batch_embeddings as $d){
                    $sub_id = $d->post_type . '_' . $d->post_id;

                    // if this item already has this embedding data calculated
                    if($calculated_phrase_data->calc_index[$sentence] >= $d->embed_index){
                        // skip to the next to save time
                        continue;
                    }

                    // if the sub item is the main item
                    if($id === $sub_id){
                        // tag the embed index so we know it's counted
                        if($calculated_phrase_data->calc_index[$sentence] < $d->embed_index){
                            $calculated_phrase_data->calc_index[$sentence] = $d->embed_index;
                        }
                        // skip to the next because we don't need to determine how related the post is to itself
                        continue;
                    }

                    // run the calculation to see how related we are
                    $calculation = self::compare_post_embeddings($phrase_object, $d);

                    // if we pass the threshold for minimum relatability
                    if($calculation > 0.45){
                        // add it to the list
                        $calculated_phrase_data->calculation[$sentence][$sub_id] = $calculation;
                    }
                    
                    if($calculated_phrase_data->calc_index[$sentence] < $d->embed_index){
                        $calculated_phrase_data->calc_index[$sentence] = $d->embed_index;
                    }

                    $calculated_phrase_data->calc_count += 1;
                }

                // save periodically to make sure that we don't lose data
                if($count > 100){
                    self::save_calculated_single_post_embedding_data($calculated_phrase_data, true);
                    $count = 0;
                }

                $count++;
            }
        }

        self::save_calculated_single_post_embedding_data($calculated_phrase_data, true);

        if($return_calculations){
            return $calculated_phrase_data->calc_count;
        }

        return $completed ? 'completed': 'uncompleted';
    }

    /**
     * Gets the calculated phrase data for a specific post.
     * Caches data between calls to cut down on DB hits
     **/
    public static function get_embedding_calc_phrases($post, $decode = false, $ignore_cache = false){
        global $wpdb;
        $calc_table = $wpdb->prefix . 'wpil_ai_embedding_phrase_calculation_data';

        if(empty($post) || !is_a($post, 'Wpil_Model_Post')){
            return array();
        }
        $pid = $post->type . '_' . $post->id;

        if(isset(self::$cached_post_sentence_embedding_data[$pid]) && !$ignore_cache){
            $data = self::$cached_post_sentence_embedding_data[$pid];
        }else{
            $data = $wpdb->get_row($wpdb->prepare("SELECT * FROM {$calc_table} WHERE `post_id` = %d AND `post_type` = %s", $post->id, $post->type));
            // if there is no data
            if(empty($data)){
                // set a flag so that we know that theres' nothing here
                $data = 'no-calculation-data';
            }

            self::$cached_post_sentence_embedding_data[$pid] = $data;
        }

        if($data === 'no-calculation-data'){
            return array();
        }

        // currently, we're decoding outside of the cache to try and save space in the cache.
        // if this gets to be a big time sinke, decompress before adding to cache
        if($decode && !empty($data)){
            if(isset($data->calculation)){
                $data->calculation = Wpil_Toolbox::json_decompress($data->calculation, true);
            }
            if(isset($data->calc_index)){
                $data->calc_index = Wpil_Toolbox::json_decompress($data->calc_index, true);
            }
        }

        return $data;
    }

    /**
     * 
     **/
    public static function has_calculated_phrase_embeddings($target_post = array()){
        if(empty($target_post) || !is_a($target_post, 'Wpil_Model_Post')){
            return false;
        }

        // get the latest index
        $last_embedding_index = self::get_last_embedding_index();

        // get the calculated phrases
        $calculated_phrase_data = self::get_embedding_calc_phrases($target_post);

        if(empty($calculated_phrase_data) || empty($calculated_phrase_data->calc_index)){
            return false;
        }

        $index = Wpil_Toolbox::json_decompress($calculated_phrase_data->calc_index);

        if(empty($index)){
            return false;
        }

        foreach($index as $calc_index){
            if($calc_index < $last_embedding_index){
                return false;
            }
        }

        return true;
    }

    /**
     * Deletes the phrase embedding data for a specific post
     * @param Wpil_Model_Post $post
     **/
    public static function clear_post_phrase_embedding_data($post = array(), $age = 0){
        global $wpdb;
        $embedding_table = $wpdb->prefix . "wpil_ai_embedding_phrase_data";
        $embedding_calc_table = $wpdb->prefix . "wpil_ai_embedding_phrase_calculation_data";

        if(empty($post) || !is_a($post, 'Wpil_Model_Post')){
            return true;
        }

        $where = "`post_id` = {$post->id} AND `post_type` = '{$post->type}'";

        if(!empty($age) && is_numeric($age)){
            $age = intval($age);
            $where .= " AND `process_time` < {$age}";
        }

        $wpdb->query("DELETE FROM {$embedding_table} WHERE $where");
        $wpdb->query("DELETE FROM {$embedding_calc_table} WHERE $where");
    }

    /**
     * Deletes post sentence embedding data so that we don't max out the user's database
     * @param Wpil_Model_Post $post
     **/
    public static function housekeep_phrase_embedding_data(){
        global $wpdb;
        $embedding_table = $wpdb->prefix . "wpil_ai_embedding_phrase_data";

        $wpdb->query("DELETE FROM {$embedding_table} WHERE `embed_index` NOT IN (
            SELECT `embed_index` FROM ( 
              SELECT `embed_index` FROM {$embedding_table} ORDER BY embed_index DESC LIMIT 10
            ) AS newest
          )"
        );
    }


    /**
     * Checks to see if there is embedding data stored
     **/
    public static function has_calculated_embedding_data(){
        global $wpdb;
        $table = $wpdb->prefix . 'wpil_ai_embedding_calculation_data';

        return !empty($wpdb->get_var("SELECT COUNT(*) FROM $table LIMIT 1"));
    }

    /**
     * Gets all of the embedding data for posts|terms that are stored in the embeddings table.
     * Can return data for a specific post|term
     **/
    public static function get_calculated_embedding_data($post_id = 0, $post_type = 'post', $offset = 0, $limit = 500){
        global $wpdb;
        $table = $wpdb->prefix . 'wpil_ai_embedding_calculation_data';
        $posts = array();

        if($post_type !== 'post' && $post_type !== 'term'){
            return $posts;
        }

        // Bulk pagination should return only the requested page and must not
        // populate the shared single-post cache, otherwise sitemap generation
        // keeps re-accumulating prior batches in memory.
        if(empty($post_id)){
            $data = $wpdb->get_results($wpdb->prepare("SELECT * FROM {$table} LIMIT %d OFFSET %d", $limit, ($limit * $offset)));
            return !empty($data) ? $data : $posts;
        }

        $search_id = $post_type . '_' . $post_id;
        // if we have a post id and there is no prior instance of this data
        if(!empty($post_id) && !isset(self::$cached_embedding_data[$search_id])){
            $data = $wpdb->get_results($wpdb->prepare("SELECT * FROM {$table} WHERE `post_id` = %d AND `post_type` = %s", $post_id, $post_type));

            if(!empty($data)){
                foreach($data as $dat){
                    $id = $dat->post_type . '_' . $dat->post_id;
                    if(!isset(self::$cached_embedding_data[$id])){
                        self::$cached_embedding_data[$id] = $dat;
                    }
                }
            }else{
                self::$cached_embedding_data[$search_id] = 'no-calculations';
            }
        }

        if(self::$cached_embedding_data === 'no-calculations'){
            return $posts;
        }

        if(!empty($post_id)){
            $id = $post_type . '_' . $post_id;
            return (isset(self::$cached_embedding_data[$id]) && !empty(self::$cached_embedding_data[$id]) && self::$cached_embedding_data[$id] !== 'no-calculations') ? self::$cached_embedding_data[$id]: array();
        }else{
            return self::$cached_embedding_data;
        }
    }

    /**
     * Gets the embedding data for a specific post from the calculation table.
     **/
    public static function get_embedding_relatedness_data($post_id = 0, $post_type = 'post', $assoc = false){
        global $wpdb;
        $table = $wpdb->prefix . 'wpil_ai_embedding_calculation_data';
        $posts = array();

        if($post_type !== 'post' && $post_type !== 'term' || empty($post_id)){
            return $posts;
        }

        $data = $wpdb->get_var($wpdb->prepare("SELECT `calculation` FROM {$table} WHERE `post_id` = %d AND `post_type` = %s", $post_id, $post_type));

        if(!empty($data)){
            $data = Wpil_Toolbox::json_decompress($data, $assoc);
            if(!empty($data) && (is_object($data) || is_array($data))){
                $posts = $data;
            }
        }

        return $posts;
    }

    /**
     * Gets the ids for posts|terms that are stored in the embeddings table
     **/
    public static function get_calculated_embedding_post_ids(){
        global $wpdb;
        $table = $wpdb->prefix . 'wpil_ai_embedding_calculation_data';
        $posts = array();

        $last_embedding_index = self::get_last_embedding_index();
        if(empty($last_embedding_index)){
            return $posts;
        }

        $embedding_data = $wpdb->get_results("SELECT `post_id`, `post_type` FROM {$table} WHERE `calc_index` >= {$last_embedding_index}");

        if(!empty($embedding_data)){
            foreach($embedding_data as $data){
                $id = $data->post_type . '_' . $data->post_id;
                $posts[$id] = true;
            }
        }

        return $posts;
    }

    public static function get_post_embedding_data($decode_embeddings = false, $index_offset = 0, $search_limit = 0){
        global $wpdb;
        $table = $wpdb->prefix . 'wpil_ai_embedding_data';

        $search_limit = (int)$search_limit;
        $limit = !empty($search_limit) ? "LIMIT {$search_limit}": "";

        $embedding_index = (int)$index_offset;
        if(!empty($embedding_index)){
            $embedding_data = $wpdb->get_results("SELECT * FROM {$table} WHERE `embed_index` > {$embedding_index} {$limit}");
        }else{
            $embedding_data = $wpdb->get_results("SELECT * FROM {$table} {$limit}");
        }
        

        if(!empty($embedding_data) && $decode_embeddings){
            foreach($embedding_data as $key => $data){
                $embedding_data[$key]->embed_data = Wpil_Toolbox::json_decompress($data->embed_data, null, true);
            }
        }

        return (!empty($embedding_data)) ? $embedding_data: array();
    }

    /**
     * Gets the embedding data for a post that has had all of it's sentences processed
     **/
    public static function get_single_post_embedding_data($post, $decode_embeddings = false){
        global $wpdb;
        $table = $wpdb->prefix . 'wpil_ai_embedding_phrase_data';

        if(empty($post) || !is_a($post, 'Wpil_Model_Post')){
            return array();
        }

        $embedding_data = $wpdb->get_results($wpdb->prepare("SELECT * FROM {$table} WHERE `post_id` = %d AND `post_type` = %s", $post->id, $post->type));

        if(!empty($embedding_data) && $decode_embeddings){
            foreach($embedding_data as $key => $data){
                $embedding_data[$key]->embed_data = Wpil_Toolbox::json_decompress($data->embed_data);
            }
        }

        return (!empty($embedding_data)) ? $embedding_data[0]: array();
    }

    public static function save_calculated_embedding_data($embedding_data = array(), $partial = false){
        global $wpdb;
        $table = $wpdb->prefix . 'wpil_ai_embedding_calculation_data';

        if(empty($embedding_data)){
            return 0;
        }

        $max_package_size = Wpil_Toolbox::get_max_allowable_package_size();
        $insert_length = 0;
        $last_embedding_index = self::get_last_embedding_index();
        $time = time();
        $total_count = 0;
        $count = 0;
        $insert_query = "INSERT INTO {$table} (post_id, post_type, data_type, calculation, calc_index, calc_count, process_time, model_version) VALUES ";
        $insert_data = array();
        $place_holders = array();
        $place_holder_string = "('%d', '%s', '%d', '%s', '%d', '%d', '%d', '%s')";
        $limit = 100;
        foreach($embedding_data as $key => $dat){
            $total_count++;
            $ids = explode('_', $key);
            if($partial){
                if (! is_array($dat->calculation)) {
                    $dat->calculation = array();
                }
                $embeddings = Wpil_Toolbox::json_compress($dat->calculation);
                $last_embedding_index = $dat->calc_index;
                $calc_count = count($dat->calculation);
                $model = $dat->model_version;
            }else{
                if (! is_array($dat['embeddings'])) {
                    $dat['embeddings'] = array();
                }
                $embeddings = Wpil_Toolbox::json_compress($dat['embeddings']);
                $calc_count = count($dat['embeddings']);
                $model = $dat['model_version'];
            }

            array_push(
                $insert_data, 
                $ids[1],
                $ids[0],
                (($ids[0] === 'post') ? 1: 0),
                $embeddings,
                $last_embedding_index,
                $calc_count,
                $time,
                $model
            );
            $place_holders[] = $place_holder_string;

            // increase the estimated insert length
            $insert_size = (strlen($embeddings) + strlen($place_holder_string) + 200); // 200 is to cover the other bits of data that are being inserted
            $insert_length += $insert_size;

            // if we've hit the limit or processing a similarly sized dataset will push us over the limit
            if($count >= $limit || (!empty($max_package_size) && ($insert_length + $insert_size) > $max_package_size)){
                // assemble the insert
                $insert = ($insert_query . implode(', ', $place_holders));
                $insert = $wpdb->prepare($insert, $insert_data);
                // insert the data
                $wpdb->query($insert);
                // reset the data variables
                $insert_data = [];
                $place_holders = [];
                $count = 0;
                $insert_length = 0;
            }

            $count++;
        }

        // if we still have data that hasn't been inserted
        if(!empty($insert_data) && !empty($place_holders)){
            // assemble the insert
            $insert = ($insert_query . implode(', ', $place_holders));
            $insert = $wpdb->prepare($insert, $insert_data);
            // and insert the data
            $wpdb->query($insert);
        }

        // if we're doing stepped saving
        if($partial){
            self::clear_duplicate_calculated_embeddings();
        }

        // return the total number of posts processed
        return $total_count;
    }

    /**
     * Removes the duplicate embedding calcs that happen when generating the calculations
     **/
    public static function clear_duplicate_calculated_embeddings($phrases = false){
        global $wpdb;
        $table = $wpdb->prefix . ((empty($phrases)) ? "wpil_ai_embedding_calculation_data": "wpil_ai_embedding_phrase_calculation_data");
        $temp_table = $wpdb->prefix . ((empty($phrases)) ? "wpil_ai_temp_calculation_data": "wpil_ai_temp_phrase_calculation_data") ;

        if(empty($wpdb->query("SHOW TABLES LIKE '{$table}'"))){
            return;
        }

        // create a temporary table with the latest records
        $wpdb->query("CREATE TEMPORARY TABLE {$temp_table} AS
        SELECT post_id, post_type, MAX(embed_index) AS max_embed_index
        FROM {$table}
        GROUP BY post_id, post_type");

        // delete records that are not the latest
        $wpdb->query("DELETE FROM {$table}
        WHERE (post_id, post_type, embed_index) NOT IN (
            SELECT post_id, post_type, max_embed_index
            FROM {$temp_table}
        )");

        // drop the temporary table
        $wpdb->query("DROP TEMPORARY TABLE {$temp_table}");
    }

    /**
     * Saves the embedding calculations for a single post's phrases
     * @param object $embedding_data
     **/
    public static function save_calculated_single_post_embedding_data($embedding_data = array()){
        global $wpdb;
        $table = $wpdb->prefix . "wpil_ai_embedding_phrase_calculation_data";

        if(empty($embedding_data)){
            return 0;
        }

        $no_data = 1;
        $phrase_string = '';
        foreach($embedding_data->calculation as $sentence => $dat){
            if(!empty($dat)){
                $no_data = 0;
            }
            $phrase_string .= $sentence;
        }

        $post = new Wpil_Model_Post($embedding_data->post_id, $embedding_data->post_type);

        $data = array(
            'post_id' => $post->id,
            'post_type' => $post->type,
            'data_type' => (($post->type === 'post') ? 1: 0),
            'post_phrase_id' => Wpil_Toolbox::create_post_content_id($post),
            'calculation' => Wpil_Toolbox::json_compress($embedding_data->calculation),
            'calc_index' => Wpil_Toolbox::json_compress($embedding_data->calc_index),
            'calc_count' => $embedding_data->calc_count,
            'no_data' => $no_data,
            'process_time' => time(),
            'model_version' => $embedding_data->model_version,
            'dimension_count' => $embedding_data->dimension_count,
        );

        $wpdb->insert($table, $data);

        if(!empty($wpdb->insert_id)){
            self::clear_duplicate_calculated_embeddings(true);
        }
    }

    /**
     * Calculates how related all the posts 
     **/
    public static function calculate_relatedness_sitemap(){
        $limit = Wpil_Settings::get_ai_sitemap_relatedness_threshold();
        $calculated = array();
        $step = 0;
        while(!Wpil_Toolbox::is_over_memory_limit() && !empty($data = self::get_calculated_embedding_data(0, 'post', $step))){
            foreach($data as $dat){
                $id = $dat->post_type . '_' . $dat->post_id;
                $calc = Wpil_Toolbox::json_decompress($dat->calculation);

                if(!isset($calculated[$id])){
                    $calculated[$id] = array();
                }

                if(!empty($calc)){
                    foreach($calc as $p_id => $c){
                        if($c >= $limit && $c < 0.999){
                            $calculated[$id][$p_id] = $c;
                        }
                    }
                }

                unset($calc);
            }

            unset($data);
            $step++;
        }

        return $calculated;
    }

    /**
     * Gets the product data
     **/
    public static function get_product_data(){
        global $wpdb;
        $table = $wpdb->prefix . "wpil_ai_product_data";
        $products = $wpdb->get_results("SELECT * FROM {$table} WHERE `product_count` > 0");
        return (!empty($products)) ? $products: array();
    }

    /**
     * Calculates products mentioned by specific posts 
     **/
    public static function calculate_product_sitemap(){
        $calculated = array();
        $data = self::get_product_data();

        foreach($data as $dat){
            $id = $dat->post_type . '_' . $dat->post_id;
            $products = Wpil_Toolbox::json_decompress($dat->products);

            if(!isset($calculated[$id])){
                $calculated[$id] = array();
            }

            if(!empty($products)){
                foreach($products as $product){
                    $p_id = trim(mb_strtolower($product));

                    if(!isset($calculated[$p_id])){
                        $calculated[$p_id] = array();
                    }

                    $calculated[$id][$p_id] = addslashes(html_entity_decode($product));
                }
            }
        }

        return $calculated;
    }

    /**
     * Export table data to CSV
     */
    public static function get_upload_file($time = '')
    {
        $dir = self::get_upload_dir();

        // if we aren't able to write to any directories
        if(empty($dir)){
            // return false
            return false;
        }

        // create the file name that we'll be working with
        if(empty($time)){
            $time = time();
        }

        $filename = 'OAI_' . $time . '_upload.jsonl';

        if(!file_exists($filename)){
            // if this is the first go round, clear any old uploads
            self::delete_upload_files();

            $fp = fopen($dir . $filename, 'w');
        } else {
            $fp = fopen($dir . $filename, 'a');
        }

        return $fp;
    }

    /**
     * Clears any OAI upload files that are present
     **/
    public static function delete_upload_files(){
        $dir = self::get_upload_dir();

        // if we aren't able to write to any directories
        if(empty($dir)){
            // return false
            return false;
        }

        $files = glob($dir . '*_upload.jsonl');
        if(!empty($files)){
            foreach($files as $file){
                unlink($file);
            }
        }
    }

    public static function get_upload_dir(){
        // get the directory that we'll be writing the export to
        $dir = false;
        if(is_writable(WP_INTERNAL_LINKING_PLUGIN_DIR)){
            if(!is_dir(WP_INTERNAL_LINKING_PLUGIN_DIR . 'includes/exports/')){
                wp_mkdir_p(WP_INTERNAL_LINKING_PLUGIN_DIR . 'includes/exports/');
            }

            // if it's possible, write to the plugin directory
            $dir = WP_INTERNAL_LINKING_PLUGIN_DIR . 'includes/exports/';
        }else{
            // if writing to the plugin directory isn't possible, try for the uploads folder
            $uploads = wp_upload_dir(null, false);
            if(!empty($uploads) && isset($uploads['basedir']) && is_writable($uploads['basedir'])){
                if(wp_mkdir_p(trailingslashit($uploads['basedir']). 'link-whisper-premium/exports')){
                    $dir = trailingslashit($uploads['basedir']). 'link-whisper-premium/exports/';
                }
            }
        }

        return $dir;
    }

    /**
     * JSON decodes the data from OpenAI responses
     **/
    public static function decode($response, $skip_recover = false, $log = false){
        $decoded = false;
        if(is_string($response) && !empty($response)){
            $decoded = json_decode($response);

            if(empty($decoded) && !$skip_recover){
                // trim the response so we don't go insane
                $response = trim($response);
                $start_marker = '```json';
                $end_marker = '```';
                $start_marker_len = strlen($start_marker);
                $end_marker_len = strlen($end_marker);
                if(0 === strpos($response, $start_marker) && (strrpos($response, $end_marker) + $end_marker_len) === strlen($response)){
                    $response = trim(substr($response, $start_marker_len, strrpos($response, $end_marker) - $start_marker_len));
                }

                $decoded = json_decode($response);

                // if that didn't work
                if(empty($decoded)){
                    // try replacing any control characters
                    $maybe_ready = mb_eregi_replace('[[:cntrl:]]', ' ', $response);
                    if(!empty($maybe_ready)){
                        $decoded = json_decode($maybe_ready);
                    }
                }

                if(empty($decoded)){
                    $decoded = json_decode(self::attempt_recover_json($response));
                }
            }
        }elseif((is_object($response) || is_array($response)) && !empty($response)){
            $decoded = $response;
        }

        return !empty($decoded) && $decoded !== false && $decoded !== null ? $decoded: false;
    }

    public static function count_tokens($text = '', $model = '', $return_tokens = false) {
        $tokens = EncodingFactory::createByModelName($model)->encode($text);
    
        return ($return_tokens) ? $tokens: count($tokens);
    }

    public static function trim_text_to_token_limit($text = '', $model = '', $limit = 0){
        $tokens = self::count_tokens($text, $model, true);

        if($limit > 0 && !empty($tokens)){
            $tokens = array_slice($tokens, 0, $limit);
        }

        return EncodingFactory::createByModelName($model)->decode($tokens);
    }

    /**
     * Breaks a post's content up into chunks without splitting words
     * @param string $content The content to be split
     * @param int $length The byte length that each chunk should be
     **/
    public static function chunk_post_content($content, $chunk_length = 0){
        if(empty($content) || empty($chunk_length)){
            return $content;
        }

        // Split the string into words and delimiters (spaces, punctuation)
        $words = preg_split('/(\s+)/u', $content, -1, PREG_SPLIT_DELIM_CAPTURE);

        $chunks = array();
        $current_chunk = '';

        foreach ($words as $word) {
            // Calculate the length of the current chunk plus the new word
            $length = mb_strlen($current_chunk . $word, 'UTF-8');
            
            if ($length <= $chunk_length) {
                // Append the word to the current chunk
                $current_chunk .= $word;
            } else {
                // If the current chunk is not empty, add it to the chunks array
                if ($current_chunk !== '') {
                    $chunks[] = $current_chunk;
                }
                // Start a new chunk with the current word
                $current_chunk = $word;

                // Handle the case where a single word exceeds 2500 characters
                if (mb_strlen($word, 'UTF-8') > $chunk_length) {
                    // Optionally split the word or handle it according to your needs
                    $chunks[] = $current_chunk;
                    $current_chunk = '';
                }
            }
        }

        // Add any remaining text in the current chunk to the chunks array
        if ($current_chunk !== '') {
            $chunks[] = $current_chunk;
        }

        return $chunks;
    }

    /**
     * Gets the list of posts that are supposed to be chunk-processed
     **/
    public static function get_chunked_posts(){
        $posts = get_transient('wpil_chunked_ai_process_posts');
        return (!empty($posts)) ? $posts: array();
    }

    /**
     * Saves a post to be chunk-processed to the chunk process list.
     * @param Wpil_Post|string A Link Whisper post object or a post_id string
     **/
    public static function save_chunked_post($post){
        if(empty($post)){
            return false;
        }

        $posts = self::get_chunked_posts();
        if(is_string($post)){
            $posts[$post] = true;
        }else{
            $id = $post->post_type . '_' . $post->post_id;
            $posts[$id] = true;
        }

        set_transient('wpil_chunked_ai_process_posts', $posts, HOUR_IN_SECONDS * 12);
        return true;
    }

    /**
     * Removes a post from the list of posts that are supposed to be chunk-processed
     * @param Wpil_Post|string A Link Whisper post object or a post_id string
     **/
    public static function remove_chunked_post($post){
        if(empty($post)){
            return false;
        }

        $posts = self::get_chunked_posts();

        if(empty($posts)){
            return true;
        }

        if(is_string($post)){
            $id = $post;
        }else{
            $id = $post->post_type . '_' . $post->post_id;
        }
        
        if(isset($posts[$id])){
            unset($posts[$id]);
            set_transient('wpil_chunked_ai_process_posts', $posts, HOUR_IN_SECONDS * 12);
        }
        
        return true;
    }

    /**
     * Gets the embedding relatedness score for specific posts
     * @param Wpil_Model_Post $post_a
     * @param Wpil_Model_Post $post_b
     **/
    public static function get_post_relationship_score($post_a = array(), $post_b = array()){
        $score = 0.000;
        if(empty($post_a) || empty($post_b)){
            return $score;
        }

        $a = self::get_calculated_embedding_data($post_a->id, $post_a->type);

        if(empty($a)){
            return $score;
        }

        $id = $post_b->type . '_' . $post_b->id;
        $calc = Wpil_Toolbox::json_decompress($a->calculation, true);

        if(!empty($calc) && isset($calc[$id])){
            $score = $calc[$id];
        }

        return floatval($score);
    }

    /**
     * Gets the embedding relatedness score for specific sentences to posts
     * @param Wpil_Model_Post $post_a the post that has the sentence that we want to score
     * @param Wpil_Model_Post $post_b the post that we want to see how related to the current sentence
     * @param string $sentence the sentence that we're checking for relatedness to post_b
     **/
    public static function get_sentence_relationship_score($post_a = array(), $post_b = array(), $sentence = ''){
        $score = 0.000;
        if(empty($post_a) || empty($post_b) || empty($sentence)){
            return $score;
        }

        $calculated_phrases = self::get_embedding_calc_phrases($post_a, true);

        if(empty($calculated_phrases)){
            return $score;
        }

        $id = $post_b->type . '_' . $post_b->id;

        if(isset($calculated_phrases->calculation[$sentence][$id])){
            $score = $calculated_phrases->calculation[$sentence][$id];
        }

        return floatval($score);
    }

    /**
     * Deletes all AI Suggestion related data for a specific post
     * @param Wpil_Model_Post $post
     **/
    public static function clear_ai_suggestion_data($post = array(), $age = 0){
        if(empty($post) || !is_a($post, 'Wpil_Model_Post')){
            return true;
        }

        self::clear_processed_anchor_sentences($post, $age);
        self::clear_post_phrase_suggestion_sentences($post, $age);
        self::clear_post_phrase_embedding_data($post, $age);
    }
    
    /**
     * Checks if post content has changed since the process was last run
     **/
    public static function sentence_post_id_changed($post = array()){
        global $wpdb;
        $table = $wpdb->prefix . "wpil_ai_embedding_phrase_data";

        if(empty($post) || !is_a($post, 'Wpil_Model_Post')){
            return true;
        }

        $phrase_id = $wpdb->get_var($wpdb->prepare("SELECT `post_phrase_id` FROM {$table} WHERE `post_id` = %d AND `post_type` = %s", $post->id, $post->type));

        // if there are no results
        if(empty($phrase_id)){
            // say that the id has changed
            return true;
        }

        // if the stored id is difference from the current id >>> return true ||| Wotyherwise, it has neot changed!
        return $phrase_id !== Wpil_Toolbox::create_post_content_id($post);
    }

    /**
     * 
     **/
    public static function delay_batch_process($process = ''){
        if(empty($process)){
            return false;
        }

        $processes = get_transient('wpil_oai_batch_process_delay');
        if(empty($processes) && !is_array($processes)){
            $processes = array($process => time() + (DAY_IN_SECONDS + HOUR_IN_SECONDS));
        }else{
            $processes[$process] = time() + (DAY_IN_SECONDS + HOUR_IN_SECONDS);
        }

        set_transient('wpil_oai_batch_process_delay', $processes, DAY_IN_SECONDS * 2);

        return true;
    }

    /**
     * Checks to see if the currently supplied process is under a delay
     **/
    public static function check_delayed_batch_process($process = ''){
        if(empty($process)){
            return false;
        }

        $processes = get_transient('wpil_oai_batch_process_delay');

        if(empty($processes) || !is_array($processes) || !isset($processes[$process])){
            return false;
        }

        // if the delay time still hasn't lapsed
        if($processes[$process] > time()){
            // say that we're delayed
            return true;
        }

        return false;
    }

    /**
     * Does final data processing actions for AI data
     **/
    public static function do_post_save_finishing(){
        $selected_processes = Wpil_Settings::get_selected_ai_batch_processes(true);
        $completed = true;

        if(in_array('keyword-detecting', $selected_processes)){
            $state = Wpil_TargetKeyword::process_ai_generated_keywords_data(array('state' => 'ai_generated_process'), microtime(true));
            $completed = ($state['state'] === 'ai_generated_process') ? false: true;
        }

        return $completed;
    }

    /**
     * 
     **/
    public static function get_api_rate_limits($endpoint = '', $non_batched = false){
        $doing_ajax = (defined('DOING_AJAX') && DOING_AJAX) ? true: false;
        $is_free = self::is_free_oai_subscription();
        $rate_limits = array();
        
        if($is_free){
            $rate_limits = array(
                'gpt-4o-mini' => 190000/2,
                'gpt-3.5-turbo' => 190000/2,
                'text-embedding-3-large' => 2800000/2
            );
        }else{
            if($doing_ajax || $non_batched){
                $rate_limits = array(
                    'gpt-4o' => 30000,
                    'gpt-4o-mini' => 185000,
                    'gpt-4.1-mini' => 185000,
                    'gpt-4.1-nano' => 185000,
                    'gpt-4-turbo' => 30000,
                    'gpt-3.5-turbo' => 185000,
                    'text-embedding-3-large' => 900000
                );
            }else{
                $rate_limits = array(
                    'gpt-4o' => 90000/2,
                    'gpt-4o-mini' => 1800000/2,
                    'gpt-4.1-mini' => 1800000/2,
                    'gpt-4.1-nano' => 1800000/2,
                    'gpt-4-turbo' => 90000/2,
                    'gpt-3.5-turbo' => 1800000/2,
                    'text-embedding-3-large' => 2800000/2
                );
            }
        }

        return (!empty($endpoint) && isset($rate_limits[$endpoint])) ? $rate_limits[$endpoint]: $rate_limits;
    }

    /**
     * 
     **/
    public static function get_live_oai_error_message(){
        $message = array(
            'title' => __('Processing Halted.', 'wpil'),
            'text'  => __("Link Whisper has processed all of the posts that it's able to, and has stopped.", 'wpil'),
            'code'  => 'unknown',
        );

        if(self::$rate_limited){
            $message['title']   = __("Unable to Complete: Rate Limiting Active", 'wpil');
            $message['text']    = sprintf(__("It seems that the API key's %s per hour has been reached, and you may need to wait for the processing limits to reset. If you haven't already, please wait an hour and then try again.", "wpil") . '<br><br>' . __("If you see this message again after waiting an hour, please wait 24 hours before restarting the process.", 'wpil'), '<a href="https://platform.openai.com/docs/guides/rate-limits/usage-tiers?context=tier-one">' . __('limit on how much data can be processed', 'wpil') .'</a>' );
            $message['code']    = 'rate_limited';
        }elseif(self::$insufficient_quota){
            $message['title']   = __("Unable to Complete: Credit Limit Reached", 'wpil');
            $message['text']    = __("It seems that the credit limit for the API key has been reached, and further processing isn't possible without more credit.", 'wpil') . '<br><br>' . __('To add more credit to the account, please go here: ', 'wpil') . '<br><br>' . '<a href="https://platform.openai.com/settings/organization/billing/overview" target="_blank">OpenAI Account Billing</a>';
            $message['code']    = 'insufficient_credits';
        }elseif(self::$invalid_api_key){
            $message['title']   = __("Unable to Complete: Invalid API Key", 'wpil');
            $message['text']    = __("It seems that there was a mistake when entering the API key, and OpenAI is rejecting our contact request.", 'wpil') . '<br><br>' . __('If you have the API key written down, please try re-entering it in the settings.', 'wpil') . '<br><br>' . sprintf(__('If you don\'t have the API key written down, please %s and enter it in the Settings.', 'wpil'), '<a href="https://platform.openai.com/api-keys" target="_blank">' . __('generate a new one from your OpenAI account', 'wpil') . '</a>');
            $message['code']    = 'invalid_api_key';
        }elseif(self::$invalid_request){
            $message['title']   = __("Unable to Complete: Invalid Request", 'wpil');
            $message['text']    = __("It seems that there was an error when reaching out to OpenAI, and post content wasn't able to be processed. Link Whisper isn't sure what caused the error, but it should be logged in the \"System Error Log\" area of the Settings.", 'wpil');
            $message['code']    = 'invalid_request';
        }else{
            $message['title']   = __("Unable to Complete: Unknown Error", 'wpil');
            $message['text']    = __('It seems that there was an error and some posts may not have been processed by OpenAI. If you see any indications that posts haven\'t been processed, please try waiting an hour and then try restarting the process.', 'wpil');
            $message['code']    = 'unknown';
        }

        //$message .= (!empty(self::$error_message)) ? "\n\n" . __('During processing, OpenAI sent along this error message: ', 'wpil') . self::$error_message . "\n\n" . __('If you need to contact support about the issue, please be sure to include this message in your ticket.', 'wpil'): '';
    
        return $message;
    }

    /**
     * 
     **/
    public static function get_linkwhisper_ai_error_message(){
        $message = array(
            'title' => __('Processing Halted.', 'wpil'),
            'text'  => __("Link Whisper has processed all of the posts that it's able to, and has stopped.", 'wpil'),
            'code'  => 'unknown',
        );

        if(self::$rate_limited){
//            $message['title']   = __("Unable to Complete: Rate Limiting Active", 'wpil');
//            $message['text']    = sprintf(__("It seems that the API key's %s per hour has been reached, and you may need to wait for the processing limits to reset. If you haven't already, please wait an hour and then try again.", "wpil") . '<br><br>' . __("If you see this message again after waiting an hour, please wait 24 hours before restarting the process.", 'wpil'), '<a href="https://platform.openai.com/docs/guides/rate-limits/usage-tiers?context=tier-one">' . __('limit on how much data can be processed', 'wpil') .'</a>' );
            $message['code']    = 'rate_limited';
        }elseif(self::$invalid_api_key){
            $message['title']   = __("Unable to Complete: API Not Accessible", 'wpil');
            $message['text']    = __("Link Whisper isn't able to make contact with the AI server. This could be caused by network traffic, or a configuration issue.", 'wpil') . '<br><br>' .__("If this is the first time this has happened, please wait 30 minutes and try again.", 'wpil') . '<br><br>' .  sprintf(__('If its happed before, please reach out to Link Whisper support %s so we can help you with this issue.', 'wpil'), '<a href="'.esc_url(WPIL_STORE_URL . '/support').'">right here</a>');
            $message['code']    = 'ai_connection';
        }elseif(self::$user_not_exist){
            $message['title']   = __("Unable to Complete: AI User Not Logged", 'wpil');
            $message['text']    = __("Unfortunately, it looks like there was an error when setting up the AI connection, and Link Whisper can't access our AI server.", 'wpil') . '<br><br>' . sprintf(__('To resolve this, please reach out to Link Whisper support %s', 'wpil'), '<a href="'.esc_url(WPIL_STORE_URL . '/support').'">right here</a>');
            $message['code']    = 'ai_user_missing';
        }elseif(self::$insufficient_quota){
            $message['title']   = __("Unable to Complete: Insufficient Credits", 'wpil');
            $message['text']    = __("Unfortunately, there aren't enough AI credits available to process the posts.", 'wpil') . '<br><br>' . __('To add more to your account, please go here: ', 'wpil') . '<br><br>' . '<a href="' .admin_url('admin.php?page=link_whisper_ai_subscription'). '" target="_blank">AI Subscription Management</a>';
            $message['code']    = 'insufficient_credits';
        }else{
            $message['title']   = __("Unable to Complete: Unknown Error", 'wpil');
            $message['text']    = __('It seems that there was an error and some posts may not have been processed by OpenAI. If you see any indications that posts haven\'t been processed, please try waiting an hour and then try restarting the process.', 'wpil');
            $message['code']    = 'unknown';
        }

        //$message .= (!empty(self::$error_message)) ? "\n\n" . __('During processing, OpenAI sent along this error message: ', 'wpil') . self::$error_message . "\n\n" . __('If you need to contact support about the issue, please be sure to include this message in your ticket.', 'wpil'): '';
    
        return $message;
    }

    /**
     * Removes the AI process streaming
     **/
    public static function disable_ai_streaming($handle_id, $content, $info){
        remove_filter('orhanerday_openai_stream_response_data', [__CLASS__, 'process_streamed_data']);
        return false;
    }

    /**
     * 
     **/
    public static function reduce_embedding_dimensions($embedding_data = array(), $dimension_count = 0){
        // Truncate the embedding to our selected number of dimensions.
        $truncated_embedding = array_slice($embedding_data, 0, $dimension_count);
        
        // Compute the L2 norm of the truncated embedding.
        $l2_norm = sqrt(array_sum(array_map(function($x) {
            return $x * $x;
        }, $truncated_embedding)));
        
        // Normalize the truncated embedding.
        if ($l2_norm != 0) {
            $normalized_embedding = array_map(function($x) use ($l2_norm) {
                return $x / $l2_norm;
            }, $truncated_embedding);
        } else {
            $normalized_embedding = $truncated_embedding;
        }

        // And return our new shortened embedding
        return $normalized_embedding;
    }

    /**
     * Checks to see if the current session has flipped a rate limiting switch
     **/
    public static function is_rate_limited(){
        return (!empty(self::$rate_limited));
    }

    /**
     * Checks to see if we're out of money
     **/
    public static function is_insufficient_quota(){
        return (!empty(self::$insufficient_quota));
    }

    /**
     * Normalizes spaces in strings to remove fun and unexpected whitespaces that OAI won't be returning to us in the output
     **/
    public static function normalize_whitespace($string = ''){
        if(!function_exists('mb_ereg_replace')){
            return preg_replace('/[^\S ]+/u', ' ', $string);
        }

        return mb_ereg_replace('( |[^\S ])+', ' ', $string);
    }

    /**
     * Estimates the cost required to process a specific piece of content.
     * Only an estimate because the output depends on ChatGPT and can't be predicted
     **/
    public static function estimate_processing_cost($content = '', $model = ''){
        $costs = self::get_standard_model_costs();
        $input_tokens = self::count_tokens($content, $model);
        $output_tokens = (!empty($input_tokens)) ? $input_tokens/10 : 0;
        $cost = 0;
        if(isset($costs[$model])){
            $cost += $costs[$model]['input'] * $input_tokens;
            $cost += $costs[$model]['output'] * $output_tokens;
        }

        return $cost;
    }

    /**
     * Estimates the token cost required to process a specific piece of content.
     * Only an estimate because the output depends on ChatGPT and can't be predicted with precision
     **/
    public static function estimate_processing_token_cost($content = '', $model = ''){
        $input_tokens = self::count_tokens($content, $model);
        $output_tokens = (!empty($input_tokens)) ? $input_tokens/10 : 0;
        $input_divisor = ($model === 'gpt-4o-mini' || $model === 'text-embedding-3-large') ? 10000: 1000;
        $output_divisor = ($model === 'gpt-4o-mini' || $model === 'text-embedding-3-large') ? 1000: 100;
        $tokens = 0;

        $tokens += ($input_tokens > 0) ? (ceil(($input_tokens/$input_divisor) + ($output_tokens/$output_divisor))): 0;

        return $tokens;
    }

    /**
     * Estimates the number of credits needed to process the site
     **/
    public static function estimate_site_processing_credit_cost($ai_linking_process_key = '', $include_ai_linking = true){
        $estimate = 0; // we're pretty much rocking 1 credit per process per post, so we'll run with that

        // get the active processes
        $selected_processes = Wpil_Settings::get_selected_ai_batch_processes(true);

        // the total number of posts that need processing
        $total_posts = self::get_total_processable_posts();

        // the number of posts that are already processed
        $processed_posts = self::get_completed_post_stats(true);

        // go over the processes
        foreach($selected_processes as $process){
            if(isset($processed_posts[$process])){
                $estimate += ($total_posts - $processed_posts[$process]);
            }else{
                $estimate += $total_posts;
            }
        }

        // if we're going to be doing post linking too
        if($include_ai_linking && (!empty($ai_linking_process_key) || null === $ai_linking_process_key)){
            if(!empty($ai_linking_process_key)){
                $remaining = Wpil_LinkMapping::get_relation_map_pending_ai_item_count($ai_linking_process_key, array(), array(), true);
                if($remaining < 1 && Wpil_LinkMapping::get_relation_map_total_item_count($ai_linking_process_key) < 1){
                    $remaining = $total_posts;
                }
            }else{
                $remaining = $total_posts;
            }

            // throw it on the stack of posts
            $estimate += ($remaining * 4);
        }

        // the result is our estimate!
        return $estimate;
    }

    /**
     * Gets the current number of credits that the user has available
     **/
    public static function get_available_ai_credits($refresh = false, $precision = false){
        if(!self::$ai_service_connected){
            return 0;
        }

        $credits = get_transient('wpil_ai_credit_balance');
        
        if(empty($credits) || $refresh){
            $stored_credits = self::get_stored_ai_credit_balance();
            $credits = self::call_linkwhisper_ai('return_credits', '', 'get-available-credits');
            if(!empty($credits)){
                $credits = round((float) $credits, 4);
                set_transient('wpil_ai_credit_balance', $credits, 60 * MINUTE_IN_SECONDS);
                self::set_stored_ai_credit_balance($credits);

                if(null !== $stored_credits && $credits > ($stored_credits + 0.00009)){
                    self::refresh_ai_credit_purchase_data();
                }
            }else{
                set_transient('wpil_ai_credit_balance', 'no-credits', 5 * MINUTE_IN_SECONDS);
                self::set_stored_ai_credit_balance(0);
            }
        }elseif($credits === 'no-credits'){
            return 0;
        }else{
            self::set_stored_ai_credit_balance($credits);
        }

        return ($precision) ? round((float) $credits, 4): (int) $credits;
    }

    /**
     * 
     **/
    public static function subtract_ai_credits($credits_spent = 0){
        // if there are no creds spent or it's somehow a negative number
        if(empty($credits_spent) || $credits_spent < 0){
            // just say everything's fine
            return true;
        }

        // get the current credit count
        $credits = self::get_available_ai_credits(false, true);

        // if they have creds
        if($credits > 0){
            // subtract from the total
            $credits = ($credits - $credits_spent);
            // and update
            set_transient('wpil_ai_credit_balance', $credits, HOUR_IN_SECONDS);
            self::set_stored_ai_credit_balance($credits);
        }
    }

    private static function get_stored_ai_credit_balance(){
        $credits = get_option('wpil_ai_last_known_credit_balance', null);
        return is_numeric($credits) ? round((float) $credits, 4) : null;
    }

    private static function set_stored_ai_credit_balance($credits = 0){
        update_option('wpil_ai_last_known_credit_balance', round(max(0, (float) $credits), 4), false);
    }

    private static function refresh_ai_credit_purchase_data(){
        $license_key = Wpil_License::getKey();
        if(empty($license_key)){
            return;
        }

        // When the balance jumps, do a quiet license check so the new purchase rows get logged too.
        Wpil_License::check($license_key, true, false);
    }

    /**
     * 
     **/
    public static function get_user_ai_subscription($reset = false){
        if(!self::$ai_service_connected){
            return null;
        }

        $subscription = get_transient('wpil_user_ai_subscription');
        if(empty($subscription) || $reset){
            $raw = wp_remote_post(WPIL_STORE_URL . '/wp-json/lwasc-checkout/v1/get-subscription', [
                'headers' => [ 'Content-Type' => 'application/json' ],
                'body'    => json_encode([ 'ai_id' => Wpil_Settings::get_linkwhisper_ai_user_id() ]),
                'timeout' => 45,
            ]);

            if(is_wp_error($raw) || empty(wp_remote_retrieve_body($raw))){
                // Network or server error — retry in 15 minutes, don't lock out for 24h
                set_transient('wpil_user_ai_subscription', 'no-subscription', 15 * MINUTE_IN_SECONDS);
                return false;
            }

            $response = json_decode(wp_remote_retrieve_body($raw));

            if(isset($response->success) && !empty($response->success) && isset($response->subscription)){
                $subscription = $response->subscription;
            }else{
                $subscription = 'no-subscription';
            }

            set_transient('wpil_user_ai_subscription', $subscription, 24 * HOUR_IN_SECONDS);
        }

        return ($subscription !== 'no-subscription') ? $subscription: false;
    }

    /**
     * 
     **/
    public static function ajax_setup_user_ai_subscription(){
        Wpil_Base::verify_nonce('setup-ai-subscription');
        $results = self::setup_user_ai_subscription(isset($_POST['recurring']) && !empty($_POST['recurring']));
        wp_send_json(['status' => $results]);
    }

    /**
     * 
     **/
    public static function setup_user_ai_subscription($recurring = false){
        if(!self::$ai_service_connected){
            return null;
        }

        // clear any balance transients that exist
        delete_transient('wpil_ai_credit_balance');

        // clear any OpenAI markers
        delete_option('wpil_is_free_ai_key');

        // get the subscription
        $subscription = self::get_user_ai_subscription(true);

        if(!empty($subscription) && empty($subscription->setting_new_sub) || (!$recurring && empty($subscription))){
            // refresh the credit count
            self::get_available_ai_credits(true);
            if($recurring){
                // note that the user has a new sub
                Wpil_Telemetry::log_event('linkwhisper_ai_plan_purchased');
            }else{
                Wpil_Telemetry::log_event('linkwhisper_ai_credits_purchased');
            }
        }

        return (!empty($subscription) && empty($subscription->setting_new_sub) || (!$recurring && empty($subscription))) ? 'subscription-setup': 'waiting-for-subscription';
    }

    /**
     * 
     **/
    public static function ajax_clear_user_ai_subscription(){
        Wpil_Base::verify_nonce('clear-ai-subscription');
        $results = self::clear_user_ai_subscription();
        wp_send_json(['status' => $results]);
    }

    /**
     * Unsets the active subscription so that the site stops showing the old subscription as the active one
     **/
    public static function clear_user_ai_subscription(){
        set_transient('wpil_user_ai_subscription', 'no-subscription', 24 * HOUR_IN_SECONDS);
        Wpil_Telemetry::log_event('linkwhisper_ai_plan_cancelled');
        return 'subscription-cleared';
    }

    /**
     * 
     **/
    public static function get_linkwhisper_ai_auth_url($return_url = null){
        $params = array(
            'target' => base64_encode(get_rest_url(null, '/' . Wpil_Rest::REST_SLUG . '/' . Wpil_Rest::AI_AUTH)),
            'return_url' => (!empty($return_url)) ? base64_encode($return_url): base64_encode(admin_url('admin.php?page=link_whisper_settings&tab=ai-settings&ai-subscription-check')),
            'site_url' => base64_encode(site_url()),
            'uid' => base64_encode(get_current_user_id())
        );
        $url = add_query_arg($params, WPIL_STORE_URL . '/connect-link-whisper-ai/');
        return $url;
    }

    /**
     * 
     **/
    public static function assemble_data_for_ai_maintenace(){
        // recommend removing low relatedness links
        // assemble infos that describe the post for ai
        // shoot off a package of that data
        // process results and package into nice form
    }

    // more functions...

    /**
     * Determines what broad niche/category the site is in
     **/
    public static function analyze_site_niche(){
        global $wpdb;

        // try getting a representative post
        $post = null;

        // ideally, the "About us" page...
        $maybe_page = $wpdb->get_col("SELECT `ID` FROM {$wpdb->posts} WHERE ");
        
    }

    /**
     * Guess the "about me/us" page.
     * Returns: ['post_id' => int|null, 'confidence' => 0..1, 'candidates' => array]
     */
    public static function guess_about_page($limit = 50){
        $exclude_slugs = [
            'contact', 'privacy-policy', 'terms', 'terms-and-conditions', 'checkout',
            'cart', 'my-account', 'account', 'shop', 'store', 'blog'
        ];

        $about_keywords = [
            'about', 'about me', 'about-me', 'about us', 'about-us',
            'bio', 'biography', 'my story', 'my-story', 'our story', 'our-story',
            'who i am', 'who-i-am', 'who we are', 'who-we-are',
            'team'
        ];

        $menu_page_ids = self::lw_collect_menu_page_ids();

        $pages = get_posts([
            'post_type'      => 'page',
            'post_status'    => 'publish',
            'numberposts'    => $limit,
            'orderby'        => 'date',
            'order'          => 'DESC',
            'fields'         => 'ids',
            'no_found_rows'  => true,
            'suppress_filters' => true,
        ]);

        $front_id  = (int) get_option('page_on_front');
        $posts_id  = (int) get_option('page_for_posts');

        $scored = [];

        foreach ($pages as $page_id) {
            if ((int)$page_id === $front_id || (int)$page_id === $posts_id) {
                continue;
            }

            $post = get_post($page_id);
            if (!$post) continue;

            $slug  = '';
            $title = '';
            if(isset($post->post_name) && !empty($post->post_name)){
                $slug  = strtolower($post->post_name);
            }

            if(isset($post->post_title) && !empty($post->post_title)){
                $title = strtolower($post->post_title);
            }


            // Exclude obvious non-about pages
            foreach ($exclude_slugs as $bad) {
                if ($slug === $bad || strpos($slug, $bad) !== false) {
                    continue 2;
                }
            }

            $content = strtolower(wp_strip_all_tags($post->post_content ?? ''));
            $score = 0;
            $reasons = [];

            // Slug and title keyword scoring
            foreach ($about_keywords as $kw) {
                $kw_norm = strtolower($kw);

                if ($kw_norm && strpos($slug, str_replace(' ', '-', $kw_norm)) !== false) {
                    $score += 50;
                    $reasons[] = "slug-match:$kw_norm";
                } elseif ($kw_norm && strpos($title, $kw_norm) !== false) {
                    $score += 35;
                    $reasons[] = "title-match:$kw_norm";
                }
            }

            // Menu presence
            if (isset($menu_page_ids[$page_id])) {
                $score += 25;
                $reasons[] = "in-menu";

                $menu_info = $menu_page_ids[$page_id];
                if (!empty($menu_info['label']) && strpos(strtolower($menu_info['label']), 'about') !== false) {
                    $score += 25;
                    $reasons[] = "menu-label-about";
                }
                if (!empty($menu_info['is_top_level'])) {
                    $score += 10;
                    $reasons[] = "menu-top-level";
                }
            }

            // Page template hint
            $tpl = (string) get_post_meta($page_id, '_wp_page_template', true);
            if ($tpl && strpos(strtolower($tpl), 'about') !== false) {
                $score += 15;
                $reasons[] = "template-about";
            }

            // Content phrase hints
            $phrases = [
                "my name is", "hi i'm", "hi im", "i am ", "about me", "about us",
                "my story", "our story", "mission", "values", "background"
            ];
            foreach ($phrases as $p) {
                if (strpos($content, $p) !== false) {
                    $score += 8;
                    $reasons[] = "content-phrase:$p";
                }
            }

            // First person ratio
            $words = preg_split('/\s+/', trim($content));
            $word_count = is_array($words) ? count($words) : 0;

            if ($word_count > 50) {
                $first_person = 0;
                $first_person += preg_match_all('/\b(i|me|my|mine|im|i\'m|ive|i\'ve)\b/', $content, $m);
                $ratio = $first_person / max(1, $word_count);

                // Scale contribution: 0.01 ratio adds a little, 0.05 adds more
                $score += (int) min(20, max(0, $ratio * 400));
                $reasons[] = "first-person-ratio:" . round($ratio, 4);
            }

            // Mild penalty if content is extremely short
            if ($word_count > 0 && $word_count < 80) {
                $score -= 10;
                $reasons[] = "very-short";
            }

            $scored[] = [
                'post_id' => (int)$page_id,
                'score'   => (int)$score,
                'title'   => $post->post_title,
                'slug'    => $post->post_name,
                'reasons' => $reasons,
            ];
        }

        usort($scored, function($a, $b){ return $b['score'] <=> $a['score']; });

        $best = $scored[0] ?? null;
        $best_score = $best['score'] ?? 0;

        // Simple confidence: compare best vs second best, and absolute score
        $second_score = $scored[1]['score'] ?? 0;
        $gap = max(0, $best_score - $second_score);

        $confidence = 0.0;
        if ($best_score >= 40) {
            $confidence = min(1.0, 0.4 + ($best_score / 200) + ($gap / 120));
        }

        return [
            'post_id'     => $best['post_id'] ?? null,
            'confidence'  => round($confidence, 3),
            'candidates'  => array_slice($scored, 0, 10),
        ];
    }

    /**
     * Collect page IDs that appear in menus, plus label and top level status.
     * Returns: [page_id => ['label' => string, 'is_top_level' => bool]]
     */
    private static function lw_collect_menu_page_ids(){
        $out = [];

        $locations = get_nav_menu_locations();
        if (empty($locations) || !is_array($locations)) return $out;

        foreach ($locations as $loc => $menu_id) {
            if (!$menu_id) continue;

            $items = wp_get_nav_menu_items($menu_id);
            if (empty($items)) continue;

            foreach ($items as $item) {
                if (empty($item->object_id) || $item->object !== 'page') continue;

                $page_id = (int) $item->object_id;
                $label = (string) (isset($item->title) && !empty($item->title) ? $item->title: '');

                // Keep the “best” label if it contains about
                $is_about_label = (stripos($label, 'about') !== false);
                $is_top_level = empty($item->menu_item_parent) || (int)$item->menu_item_parent === 0;

                if (!isset($out[$page_id])) {
                    $out[$page_id] = ['label' => $label, 'is_top_level' => $is_top_level];
                } else {
                    if ($is_about_label) $out[$page_id]['label'] = $label;
                    $out[$page_id]['is_top_level'] = $out[$page_id]['is_top_level'] || $is_top_level;
                }
            }
        }

        return $out;
    }

    /**
     * Gets the linking recommendations for a specfic post.
     * 
     * @param $post_id the ide of the post that we're linking from
     * @param $process_key The id of the mapp that we're pulling the data from
     * @param $direction are these inbound or outbound links? Default, outbound
     **/
    public static function create_link_suggestions_from_map($post_id, $process_key = '', $direction = 'outbound'){
        if(empty($post_id) || empty($process_key)){
            return false;
        }

        $pid = self::normalize_pid($post_id);
        $pid_parts = self::parse_pid($pid);
        if(empty($pid_parts['id'])){
            return false;
        }

        $post_id = $pid_parts['id'];
        $post_type = $pid_parts['type'];

        // dashboard fixes should treat money pages as inbound-only so we don't send links out from them
        if($direction === 'outbound' && self::should_skip_dashboard_fix_money_page_source($pid, $process_key)){
            return false;
        }

        $work_scope = '';
        if($process_key === md5('link-coverage-search') && in_array($direction, array('outbound', 'inbound'), true)){
            $work_scope = Wpil_LinkMapping::normalize_relation_work_scope($direction);
        }

        $map_item = Wpil_LinkMapping::get_relation_map_item($process_key, $post_id, $post_type, true, false, $work_scope);
        $local_model = 'gpt-4o-mini';

        if( empty($map_item) || 
            !isset($map_item['related_posts']) || 
            empty($map_item['related_posts'])
        ){
            return false;
        }

        $relations = $map_item['related_posts'];
        $data = [];
        $input = [];

        // get the post that we'll be focussing on
        $post = new Wpil_Model_Post($post_id, $post_type);

        // make sure the current post has a title
        if(empty($post->getTitle())){
            return false; /// exist if it doesn't
        }

        if($direction === 'outbound' && self::has_reached_ai_outbound_processing_limit($post)){
            return false;
        }

        // if we're creating links in this post pointing to other posts
        if($direction === 'outbound'){
            // check to make sure that we're reasonably sure that we can create links in the post
            $phrases = Wpil_Suggestion::getPhrases($post->getContent(), false, array(), false, array(), true);
            if(empty($phrases)){
                return false; // if we can't, oh darn!
            }

            // get the post's current links
            // if we're preventing twoway linking
            if(get_option('wpil_prevent_two_way_linking', false)){
                // get the inbound && the outbound internal links
                $post_links = Wpil_Post::getLinkedPostIDs($post, true);
            }else{
                $report_links = Wpil_Report::getReportOutboundLinks($post)['internal'];
                $post_links = [];

                if(!empty($report_links)){
                    foreach($report_links as $dat){
                        if(empty($dat->post) || empty($dat->post->id)){
                            continue;
                        }
                        $post_links[] = $dat->post->id;
                    }
                }
            }


            // get the post's keywrods
            $post_keywords = Wpil_TargetKeyword::get_active_keyword_list($post_id, $post_type);

            // set the origin post id
            self::$origin_post = $post;

            // get the keywords for the posts that we'll be mapping to
            $target_data = [];
            foreach($relations as $relation_pid){
                $relation_parts = self::parse_pid($relation_pid);
                if(empty($relation_parts['id'])){
                    continue;
                }
                $target_data[$relation_parts['type'] . '_' . $relation_parts['id']] = [
                    'id' => $relation_parts['id'],
                    'type' => $relation_parts['type'],
                    'keywords' => Wpil_TargetKeyword::get_active_keyword_list($relation_parts['id'], $relation_parts['type'])
                ];
            }

            $phrases = self::filter_ai_source_phrases_for_keyword_cannibalization($phrases, $post, array_keys($target_data));
            $content = self::get_ai_source_content_from_phrases($phrases);

            if(empty($content)){
                return false;
            }

            // assemble the data object
            $input['source'] = ['content' => self::trim_text_to_token_limit($content, $local_model, 20000), 'keywords' => $post_keywords];
            $input['targets'] = [];
            $token_size = 300 + self::count_tokens($input['source']['content'] . implode(',', $input['source']['keywords']), $local_model);
            $limit = 100000;
            $target_limit = 15;
            foreach($target_data as $target_pid => $target){
                if(count($input['targets']) > $target_limit){
                    break;
                }

                if(in_array($target['id'], $post_links)){
                    continue;
                }

                $tp = new Wpil_Model_Post($target['id'], $target['type']);

                $title = $tp->getTitle();
                if(empty($title)){
                    continue;
                }

                $dat = [
                    'title' => $title,
                    'keywords' => !empty($target['keywords']) ? $target['keywords']: [],
                    'target_id' => $tp->id,
                    'target_type' => $tp->type
                ];

                $query = self::count_tokens(($dat['title'] . ((!empty($dat['keywords']) && is_array($dat['keywords'])) ? implode(',', $dat['keywords']): '') . $dat['target_id']), $local_model);
                if($token_size + $query < $limit){
                    $token_size += $query;
                    $input['targets'][] = $dat;
                }else{
                    break;
                }
            }

            // if there are no viable targets
            if(empty($input['targets'])){
                return false; // exist
            }

            // json encode the whole thinkg
            $input = json_encode($input);

            // if there was an error
            if(empty($input) || !empty(json_last_error())){
                return false; // todo: maybe consider some error logging here!
            }

            // log the query id
            self::$query_ids[] = $post->get_pid();

            // send off our carefully assembled data!
            $data = self::determine_linking_candidates($input, $direction, $process_key);
            // and save the results!
            self::save_ai_outbound_linking_suggestions($data, $post, $process_key);
        }elseif($direction === 'inbound'){ // if we're trying to point links to the current post
            // get the existing links
            $internal_links = Wpil_Post::getLinkedPostIDs($post, true);

            // get the post's keywrods
            $post_keywords = Wpil_TargetKeyword::get_active_keyword_list($post_id, $post_type);

            // get the keywords for the posts that we'll be mapping to
            // limit
            $limit = 6;

            // assemble the data objects
            foreach($relations as $relation_pid){
                if(count($input) > $limit){
                    break;
                }

                $relation_parts = self::parse_pid($relation_pid);
                if(empty($relation_parts['id']) || in_array($relation_parts['id'], $internal_links)){
                    continue;
                }

                $source_pid = $relation_parts['type'] . '_' . $relation_parts['id'];
                if(self::should_skip_dashboard_fix_money_page_source($source_pid, $process_key)){
                    continue;
                }

                $tp = new Wpil_Model_Post($relation_parts['id'], $relation_parts['type']);
                if(self::has_reached_ai_outbound_processing_limit($tp)){
                    continue;
                }

                // check to make sure that we're reasonably sure that we can create links in the post
                $phrases = Wpil_Suggestion::getPhrases($tp->getContent(), false, array(), false, array(), true);
                $phrases = self::filter_ai_source_phrases_for_keyword_cannibalization($phrases, $tp, array($post->get_pid()));
                $content = self::get_ai_source_content_from_phrases($phrases);

                if(empty($content)){
                    continue; // if we can't, skip to the next
                }

                $kwords = Wpil_TargetKeyword::get_active_keyword_list($tp->id, $tp->type);

                $dat = json_encode([
                    'source' => [
                        'content' => self::trim_text_to_token_limit($content, 'gpt-4o-mini', 25000),
                        'keywords' => (!empty($kwords) ? $kwords: [])
                    ],
                    'target' => [
                        'title' => $post->getTitle(),
                        'keywords' => !empty($post_keywords) ? $post_keywords: [],
                        'target_id' => $post->id,
                        'target_type' => $post->type
                    ]
                ]);

                // if there was an error
                if(empty($dat) || !empty(json_last_error())){
                    // skip to hte next post
                    continue; // todo: maybe consider some error logging here!
                }

                self::$query_ids[] = $tp->get_pid();
                $input[] = $dat;
            }

            // if there is no input
            if(empty($input)){
                return false; // todo: maybe consider some error logging here!
            }

            // send off our carefully assembled data!
            $data = self::determine_linking_candidates($input, $direction, $process_key);
        }

        return $data;
    }
    
    /**
     * Uses teh spooky power of AI to determine which and wehere links should be made!
     **/
    public static function determine_linking_candidates($data = array(), $direction = 'inbound', $process_key = ''){
        global $wpdb;

        // get the max outbound links for a post
        $max_outbound = (int)get_option('wpil_max_links_per_post', 0);

        // get the max inbound links for a post
        $max_inbound = (int)get_option('wpil_max_inbound_links_per_post', 0);

        // get if we're allowed to rewrite sentences for better fits
        $rewrite_sentences = 0; // TODO: implement a setting later

        /*
        basic structure
        $data[
            source => [
                'content' => '',
                'keywords' => []
            ]        
            targets[
                [
                    'title' => '',
                    'keywords' => []
                ],
                [
                    'title' => '',
                    'keywords' => []
                ],
                [
                    'title' => '',
                    'keywords' => []
                ],
            ]
        ]

        */

        // and call!
        self::$purpose = ($direction === 'outbound') ? 'assess-outbound-links': 'assess-inbound-links';
        $prev_process_key = self::$active_linking_process_key;
        self::$active_linking_process_key = (string) $process_key;
        try{
            $results = self::call_linkwhisper_ai($data, 'gpt-5-mini', '', ['rewrite_sentences' => $rewrite_sentences, 'max_outbound_links' => $max_outbound, 'max_inbound_links' => $max_inbound]);
        }finally{
            self::$active_linking_process_key = $prev_process_key;
        }

        // return the fruits of our labours!
        return $results;
    }

    private static function normalize_pid($pid){
        if(empty($pid)){
            return '';
        }

        if(is_numeric($pid)){
            return 'post_' . (int)$pid;
        }

        if(is_string($pid) && strpos($pid, '_') !== false){
            return $pid;
        }

        return '';
    }

    public static function normalize_pid_list($ids = []){
        if(empty($ids)){
            return [];
        }

        if(!is_array($ids)){
            $ids = [$ids];
        }

        $out = [];
        foreach($ids as $id){
            $pid = self::normalize_pid($id);
            if(!empty($pid)){
                $out[] = $pid;
            }
        }

        return array_values(array_unique($out));
    }

    private static function parse_pid($pid){
        $out = [
            'type' => 'post',
            'id' => 0,
            'pid' => ''
        ];

        $pid = self::normalize_pid($pid);
        if(empty($pid)){
            return $out;
        }

        $bits = explode('_', $pid, 2);
        if(count($bits) === 2){
            $type = $bits[0];
            $id = (int)$bits[1];
            if($type === 'term'){
                $out['type'] = 'term';
            }
            $out['id'] = $id;
            $out['pid'] = $out['type'] . '_' . $id;
        }

        return $out;
    }

    private static function parse_linking_target($result){
        $target_type = '';
        $target_id = 0;

        if(is_object($result)){
            if(isset($result->target_type)){
                $target_type = (string)$result->target_type;
            }
            if(isset($result->target_id)){
                $target_id = $result->target_id;
            }
        }else{
            $target_id = $result;
        }

        if(is_string($target_id) && strpos($target_id, '_') !== false){
            $parts = self::parse_pid($target_id);
            if(!empty($parts['id'])){
                $target_id = $parts['id'];
                $target_type = $parts['type'];
            }
        }

        if(empty($target_type)){
            $target_type = 'post';
        }

        return [
            'id' => (int)$target_id,
            'type' => $target_type,
            'data_type' => ($target_type === 'post') ? 1 : 0
        ];
    }

    /**
     * Retuns a list of the currently available Dashboartd fix with ai actions
     **/
    public static function get_dashboard_fix_process_keys(){
        return array(
            md5('orphan-post-search'),
            md5('link-coverage-search'),
            md5('link-quality-search'),
            md5('broken-link-search'),
            md5('external-focus-search'),
        );
    }

    /**
     * Returns true when the process key belongs to orphaned-post fixing.
     **/
    private static function is_orphan_fix_process_key($process_key = ''){
        $process_key = is_string($process_key) ? sanitize_text_field($process_key) : '';
        if($process_key === ''){
            return false;
        }

        return ($process_key === md5('orphan-post-search'));
    }

    /**
     * Lets us tell when a process came from one of the dashboard fix tools.
     **/
    private static function is_dashboard_fix_process_key($process_key = ''){
        $process_key = is_string($process_key) ? sanitize_text_field($process_key) : '';
        if($process_key === ''){
            return false;
        }

        return in_array($process_key, self::get_dashboard_fix_process_keys(), true);
    }

    /**
     * Checks if the pid belongs to one of the pages we want to keep inbound-only.
     **/
    private static function is_money_page_pid($pid = ''){
        $parts = self::parse_pid($pid);
        if(empty($parts['id'])){
            return false;
        }

        return in_array($parts['pid'], Wpil_Settings::get_money_page_pid_list(true), true);
    }

    /**
     * Dashboard fix runs can still target money pages, but they shouldn't use them as source posts.
     **/
    private static function should_skip_dashboard_fix_money_page_source($pid = '', $process_key = ''){
        return (self::is_dashboard_fix_process_key($process_key) && self::is_money_page_pid($pid));
    }

    /**
     * Pulls the sentence text out of the phrase objects so we can hand it off to the AI.
     **/
    private static function get_ai_source_content_from_phrases($phrases = array()){
        if(empty($phrases) || !is_array($phrases)){
            return '';
        }

        $sentences = array();
        foreach($phrases as $phrase){
            if(!is_object($phrase)){
                continue;
            }

            $sentence = '';
            if(isset($phrase->sentence_src) && !empty($phrase->sentence_src)){
                $sentence = $phrase->sentence_src;
            }elseif(isset($phrase->text) && !empty($phrase->text)){
                $sentence = $phrase->text;
            }

            if(!empty($sentence)){
                $sentences[] = $sentence;
            }
        }

        return !empty($sentences) ? implode('|||', $sentences): '';
    }

    /**
     * When the setting is on, strip out any source sentences that are trying to rank another post's keyword.
     * We still keep sentences that are talking about the current post or the target post we're linking to.
     **/
    private static function filter_ai_source_phrases_for_keyword_cannibalization($phrases = array(), $source_post = null, $allowed_target_pids = array()){
        if(empty($phrases) || !is_array($phrases) || !is_a($source_post, 'Wpil_Model_Post') || !Wpil_Settings::get_prevent_keyword_cannibalization()){
            return $phrases;
        }

        $keyword_data = self::get_keyword_cannibalization_keyword_data();
        if(empty($keyword_data['keywords'])){
            return $phrases;
        }

        $allowed_lookup = array();
        $allowed_lookup[$source_post->get_pid()] = true;
        if(!empty($allowed_target_pids)){
            foreach($allowed_target_pids as $pid){
                $parts = self::parse_pid($pid);
                if(empty($parts['id'])){
                    continue;
                }

                $allowed_lookup[$parts['pid']] = true;
            }
        }

        $filtered = array();
        foreach($phrases as $phrase){
            if(!is_object($phrase)){
                continue;
            }

            $sentence = '';
            if(isset($phrase->sentence_src) && !empty($phrase->sentence_src)){
                $sentence = $phrase->sentence_src;
            }elseif(isset($phrase->text) && !empty($phrase->text)){
                $sentence = $phrase->text;
            }

            if(empty($sentence)){
                continue;
            }

            $matched_owner_pids = self::get_ai_sentence_keyword_owner_pids($sentence, $keyword_data['keywords'], $keyword_data['more_specific_keywords']);
            if(empty($matched_owner_pids)){
                $filtered[] = $phrase;
                continue;
            }

            $keep_phrase = true;
            foreach($matched_owner_pids as $pid => $matched){
                if(empty($allowed_lookup[$pid])){
                    $keep_phrase = false;
                    break;
                }
            }

            if($keep_phrase){
                $filtered[] = $phrase;
            }
        }

        return $filtered;
    }

    /**
     * Loads the active keyword data once so we don't keep hammering the DB on every AI call.
     **/
    private static function get_keyword_cannibalization_keyword_data(){
        if(!is_null(self::$keyword_cannibalization_keywords)){
            return self::$keyword_cannibalization_keywords;
        }

        $keywords = Wpil_TargetKeyword::get_all_active_keywords([],[],['ai-generated-keyword']);

        $formatted_keywords = array();
        if(!empty($keywords)){
            foreach($keywords as $keyword){
                $formatted = is_a($keyword, 'Wpil_Model_Keyword') ? $keyword: new Wpil_Model_Keyword($keyword);
                if(empty($formatted->post_id) || empty($formatted->keywords)){
                    continue;
                }

                $formatted_keywords[] = $formatted;
            }
        }

        self::$keyword_cannibalization_keywords = array(
            'keywords' => $formatted_keywords,
            'more_specific_keywords' => []//!empty($formatted_keywords) ? Wpil_Suggestion::getMoreSpecificKeywords($formatted_keywords, $formatted_keywords): array(),
        );

        return self::$keyword_cannibalization_keywords;
    }

    /**
     * Mirrors the regular suggestion matching so the AI flow respects the same target keyword guard rails.
     **/
    private static function get_ai_sentence_keyword_owner_pids($sentence = '', $target_keywords = array(), $more_specific_keywords = array()){
        if(empty($sentence) || empty($target_keywords)){
            return array();
        }

        $matched_posts = array();
        $sentence = Wpil_Word::strtolower($sentence);
        $stemmed_sentence = Wpil_Word::getStemmedSentence($sentence);
        $normalized_sentence = Wpil_Word::getStemmedSentence(Wpil_Word::remove_accents($stemmed_sentence), true);

        foreach($target_keywords as $keyword){
            if(empty($keyword) || empty($keyword->post_id) || empty($keyword->keywords) || 3 > strlen($keyword->keywords) || $keyword->keyword_type === 'ai-generated-keyword'){
                continue;
            }

            if(!self::ai_sentence_has_target_keyword_match($sentence, $stemmed_sentence, $normalized_sentence, $keyword)){
                continue;
            }

            if(isset($more_specific_keywords[$keyword->stemmed])){
                $skip_keyword = false;
                foreach($more_specific_keywords[$keyword->stemmed] as $specific_keyword){
                    if(self::ai_sentence_has_target_keyword_match($sentence, $stemmed_sentence, $normalized_sentence, $specific_keyword)){
                        $skip_keyword = true;
                        break;
                    }
                }

                if($skip_keyword){
                    continue;
                }
            }

            $matched_posts[(($keyword->post_type === 'term') ? 'term_' : 'post_') . (int) $keyword->post_id] = true;
        }

        return $matched_posts;
    }

    /**
     * Checks the sentence against a keyword using the same gentle matching rules as the normal suggestion builder.
     **/
    private static function ai_sentence_has_target_keyword_match($sentence = '', $stemmed_sentence = '', $normalized_sentence = '', $target_keyword = null){
        if(empty($target_keyword) || empty($target_keyword->keywords) || empty($target_keyword->stemmed)){
            return false;
        }

        $keyword = Wpil_Word::strtolower($target_keyword->keywords);
        $in_unstemmed = false !== strpos($sentence, $keyword);
        $in_stemmed = false !== strpos($stemmed_sentence, $target_keyword->stemmed);
        $in_normalized = false;

        if(!$in_unstemmed && !$in_stemmed){
            $in_normalized = !empty($normalized_sentence) && !empty($target_keyword->normalized) && false !== strpos($normalized_sentence, $target_keyword->normalized);
        }

        if(!$in_unstemmed && !$in_stemmed && !$in_normalized){
            return false;
        }

        if($in_unstemmed && !$in_stemmed){
            $pos = Wpil_Word::mb_strpos($sentence, $keyword);
            if(Wpil_Keyword::isPartOfWord($sentence, $keyword, $pos)){
                return false;
            }
        }

        if($in_stemmed){
            $pos = Wpil_Word::mb_strpos($stemmed_sentence, $target_keyword->stemmed);
            if(Wpil_Keyword::isPartOfWord($stemmed_sentence, $target_keyword->stemmed, $pos)){
                return false;
            }
        }

        if($in_normalized){
            $pos = Wpil_Word::mb_strpos($normalized_sentence, $target_keyword->normalized);
            if(Wpil_Keyword::isPartOfWord($normalized_sentence, $target_keyword->normalized, $pos)){
                return false;
            }
        }

        return true;
    }

    /**
     * Once a post hits the outbound cap, leave it alone for AI outbound work.
     **/
    private static function has_reached_ai_outbound_processing_limit($post = null){
        if(!is_a($post, 'Wpil_Model_Post')){
            return false;
        }

        $limit = (int) Wpil_Settings::get_ai_suggestion_outbound_limit();
        if($limit < 1){
            return false;
        }

        return ($post->getOutboundInternalLinks(true) >= $limit);
    }

    /**
     * Live orphan check for target items used in AI suggestions.
     **/
    private static function is_orphaned_target_for_ai_suggestion($target_id = 0, $target_type = 'post'){
        $target_id = (int) $target_id;
        $target_type = ($target_type === 'term') ? 'term' : 'post';
        if($target_id <= 0){
            return false;
        }

        $target_state = self::get_live_target_state($target_id, $target_type);

        return !empty($target_state['orphan_eligible']);
    }

    /**
     * Gets the current live link state for a target post/term.
     **/
    private static function get_live_target_state($target_id = 0, $target_type = 'post'){
        $target_id = (int) $target_id;
        $target_type = ($target_type === 'term') ? 'term' : 'post';
        $state = array(
            'inbound_internal' => 0,
            'outbound_internal' => 0,
            'outbound_external' => 0,
            'orphan_eligible' => false,
        );

        if($target_id <= 0){
            return $state;
        }

        $target_post = new Wpil_Model_Post($target_id, $target_type);
        if(empty($target_post) || !$target_post->check_if_post_exists()){
            return $state;
        }

        $state['inbound_internal'] = (int) $target_post->getInboundInternalLinks(true);
        $state['outbound_internal'] = (int) $target_post->getOutboundInternalLinks(true);
        $state['outbound_external'] = (int) $target_post->getOutboundExternalLinks(true);
        $state['orphan_eligible'] = ($state['inbound_internal'] <= 0);

        return $state;
    }

    /**
     * Builds the review modal display data for a post or term.
     **/
    private static function get_review_post_display_data($post = null, $ai_relatedness = ''){
        $data = array(
            'title' => '',
            'type' => '',
            'taxonomy' => '',
            'categories' => '',
            'tags' => '',
            'inbound_internal' => 0,
            'outbound_internal' => 0,
            'outbound_external' => 0,
            'post_id' => 0,
            'language' => '',
            'view_link' => '',
            'ai_relatedness' => $ai_relatedness
        );

        if(empty($post) || !is_a($post, 'Wpil_Model_Post') || !$post->check_if_post_exists()){
            return $data;
        }

        $categories = '';
        $tags = '';
        if($post->type === 'post'){
            $taxonomies = Wpil_Settings::getTermTypes();
            $terms = get_terms(array(
                'taxonomy' => $taxonomies,
                'hide_empty' => false,
                'object_ids' => $post->id,
            ));

            $categories_list = array();
            $tags_list = array();
            if(!is_wp_error($terms) && !empty($terms)){
                foreach($terms as $term){
                    $taxonomy = get_taxonomy($term->taxonomy);
                    if($taxonomy && $taxonomy->hierarchical){
                        $categories_list[] = $term->name;
                    }else{
                        $tags_list[] = $term->name;
                    }
                }
            }

            if(!empty($categories_list)){
                $categories = implode(', ', $categories_list);
            }
            if(!empty($tags_list)){
                $tags = implode(', ', $tags_list);
            }
        }

        $state = self::get_live_target_state((int) $post->id, (string) $post->type);
        $view_link = $post->getLinks()->view;
        if(!empty($view_link)){
            $view_link = Wpil_Link::filter_staging_to_live_domain($view_link);
        }

        $data = array(
            'title' => $post->getTitle(),
            'type' => $post->getType(),
            'taxonomy' => ($post->type === 'term') ? $post->getRealType() : '',
            'categories' => $categories,
            'tags' => $tags,
            'inbound_internal' => (int) $state['inbound_internal'],
            'outbound_internal' => (int) $state['outbound_internal'],
            'outbound_external' => (int) $state['outbound_external'],
            'post_id' => (int) $post->id,
            'language' => Wpil_Post::getPostLanguageCode($post),
            'view_link' => $view_link,
            'ai_relatedness' => $ai_relatedness
        );

        return $data;
    }

    /**
     * Checks if the suggestion's source now links to the target in the live report data.
     **/
    private static function did_suggestion_create_live_link($suggestion){
        if(empty($suggestion) || !is_object($suggestion) || empty($suggestion->post_id) || empty($suggestion->post_type) || empty($suggestion->target_id) || empty($suggestion->target_type)){
            return false;
        }

        $source_post = new Wpil_Model_Post((int) $suggestion->post_id, (string) $suggestion->post_type);
        if(empty($source_post) || !$source_post->check_if_post_exists()){
            return false;
        }

        $outbound_links = Wpil_Report::link_table_is_created() ? Wpil_Report::getReportOutboundLinks($source_post) : Wpil_Report::getOutboundLinks($source_post);
        $internal_links = (!empty($outbound_links['internal']) && is_array($outbound_links['internal'])) ? $outbound_links['internal'] : array();
        if(empty($internal_links)){
            return false;
        }

        foreach($internal_links as $link){
            if(empty($link->post) || empty($link->post->id) || empty($link->post->type)){
                continue;
            }

            if((int) $link->post->id === (int) $suggestion->target_id && (string) $link->post->type === (string) $suggestion->target_type){
                return true;
            }
        }

        return false;
    }

    /**
     * Applies a suggestion and reports what actually happened.
     **/
    private static function apply_linking_suggestion($suggestion, $ignore_on_fail = false){
        global $wpdb;
        $linking_table = $wpdb->prefix . 'wpil_ai_linking';

        $response = array(
            'applied' => false,
            'stale' => false,
            'message' => '',
            'reason' => '',
            'process_key' => (!empty($suggestion->process_key)) ? (string) $suggestion->process_key : '',
            'link_id' => (!empty($suggestion->ai_index)) ? (int) $suggestion->ai_index : 0,
        );

        if(empty($suggestion) || !is_object($suggestion) || empty($suggestion->ai_index)){
            $response['message'] = 'Suggestion not found.';
            $response['reason'] = 'missing_suggestion';
            return $response;
        }

        $process_key = !empty($suggestion->process_key) ? (string) $suggestion->process_key : '';
        if(self::is_orphan_fix_process_key($process_key)){
            $target_state = self::get_live_target_state((int) $suggestion->target_id, (string) $suggestion->target_type);
            if(empty($target_state['orphan_eligible'])){
                $wpdb->update($linking_table, ['ignored' => 1], ['ai_index' => (int) $suggestion->ai_index]);
                $response['stale'] = true;
                $response['message'] = 'The target already has inbound links.';
                $response['reason'] = 'target_not_orphaned';
                return $response;
            }
        }

        $target_link = self::build_target_link_from_suggestion($suggestion);
        if(empty($target_link)){
            if($ignore_on_fail){
                $wpdb->update($linking_table, ['ignored' => 1], ['ai_index' => (int) $suggestion->ai_index]);
            }

            $response['message'] = 'Unable to build the proposed link from the current suggestion data.';
            $response['reason'] = 'invalid_target_link';
            return $response;
        }

        $meta = [[
            'id' => $suggestion->target_id,
            'type' => $suggestion->target_type,
            'post_origin' => 'internal',
            'site_url' => '',
            'sentence' => $suggestion->sentence_text,
            'sentence_with_anchor' => '',
            'custom_sentence' => $target_link,
        ]];

        Wpil_Base::clear_tracked_action('link_inserted');

        if(!empty($suggestion->post_type) && $suggestion->post_type === 'term'){
            Wpil_Toolbox::update_encoded_term_meta($suggestion->post_id, 'wpil_links', $meta);
            Wpil_Term::addLinksToTerm($suggestion->post_id);
        }else{
            Wpil_Post::addLinksToContent(null, ['ID' => $suggestion->post_id], array(), false, true, $meta);
        }

        $applied = Wpil_Base::action_happened('link_inserted') || self::did_suggestion_create_live_link($suggestion);
        Wpil_Base::clear_tracked_action('link_inserted');

        if($applied){
            self::mark_suggestion_inserted_and_ignore_sentence_matches($suggestion);
            $response['applied'] = true;
            $response['message'] = 'Link inserted.';
            $response['reason'] = 'inserted';
            return $response;
        }

        if($ignore_on_fail){
            $wpdb->update($linking_table, ['ignored' => 1], ['ai_index' => (int) $suggestion->ai_index]);
        }

        $response['message'] = 'Unable to insert the link into the current content.';
        $response['reason'] = 'insert_failed';
        return $response;
    }

    /**
     * 
     **/
    public static function ignore_linking_suggestions($ids = []){
        global $wpdb;
        $linking_table = $wpdb->prefix . 'wpil_ai_linking';

        if(empty($ids)){
            return false;
        }

        if(!is_array($ids)){
            $ids = array($ids);
        }

        foreach($ids as $id){
            $wpdb->update($linking_table, ['ignored' => 1], ['ai_index' => $id]);
        }
    }

    /**
     * Creates links from the database
     **/
    public static function create_links_from_linking_suggestions($ids = []){
        global $wpdb;
        $linking_table = $wpdb->prefix . 'wpil_ai_linking';

        if(empty($ids)){
            return array(
                'applied' => false,
                'stale' => false,
                'message' => 'No suggestions were supplied.',
                'reason' => 'missing_ids',
                'process_key' => '',
                'link_id' => 0,
            );
        }

        $ids = implode(',', $ids);

        $data = $wpdb->get_results("SELECT * FROM {$linking_table} WHERE `ai_index` in ({$ids}) AND `ignored` < 1 AND `inserted` < 1");

        if(empty($data)){
            return array(
                'applied' => false,
                'stale' => false,
                'message' => 'Suggestion not found or already processed.',
                'reason' => 'missing_suggestion',
                'process_key' => '',
                'link_id' => 0,
            );
        }

        $links_inserted = 0;
        $results = array();
        foreach($data as $dat){
            if(Wpil_Base::overTimeLimit(5,30)){
                break;
            }

            $result = self::apply_linking_suggestion($dat, true);
            $results[(int) $dat->ai_index] = $result;
            if(!empty($result['applied'])){
                $links_inserted++;
            }
        }

        if(!empty($links_inserted)){
            Wpil_Telemetry::log_event('linkwhisper_ai_link_inserted', array('count' => $links_inserted));
        }

        if(count($results) === 1){
            return reset($results);
        }

        return $results;
    }

    /**
     * Automatically inserts links from suggestions
     **/
    public static function auto_create_links_from_suggestions($source_post_id = 0, $source_post_type = '', $process_key = ''){
        global $wpdb;
        $linking_table = $wpdb->prefix . 'wpil_ai_linking';

        $source_post_id = (int) $source_post_id;
        $source_post_type = ($source_post_type === 'term') ? 'term' : (($source_post_type === 'post') ? 'post' : '');
        $process_key = is_string($process_key) ? sanitize_text_field($process_key) : '';

        $where = "WHERE `ignored` < 1 AND `inserted` < 1";
        if(!empty($process_key)){
            $where .= $wpdb->prepare(" AND `process_key` = %s", $process_key);
        }
        if($source_post_id > 0){
            $where .= $wpdb->prepare(" AND `post_id` = %d", $source_post_id);
            if(!empty($source_post_type)){
                $where .= $wpdb->prepare(" AND `post_type` = %s", $source_post_type);
            }
        }

        $data = $wpdb->get_results("SELECT * FROM {$linking_table} {$where} ORDER BY COALESCE(ai_relation_score, 0) DESC, ai_index ASC LIMIT 50");

        if(empty($data)){
            return true; // return true for the benefit of whoever is listening
        }

        // get the max outbound links for a post
        $max_outbound = (int)get_option('wpil_max_links_per_post', 0);

        // get the max inbound links for a post
        $max_inbound = (int)get_option('wpil_max_inbound_links_per_post', 0);

        // get the minimum relatedness score required to auto-insert
        $auto_insert_threshold = Wpil_Settings::get_ai_auto_insert_relatedness_threshold();

        $links_inserted = 0;
        foreach($data as $dat){
            if(Wpil_Base::overTimeLimit(5,60)){
                break;
            }

            $row_process_key = !empty($dat->process_key) ? (string)$dat->process_key : $process_key;
            $orphan_only_targets = self::is_orphan_fix_process_key($row_process_key);
            if($orphan_only_targets && !self::is_orphaned_target_for_ai_suggestion((int)$dat->target_id, (string)$dat->target_type)){
                $wpdb->update($linking_table, ['ignored' => 1], ['ai_index' => $dat->ai_index]);
                continue;
            }

            if($auto_insert_threshold > 0){
                $ai_score = isset($dat->ai_relation_score) ? floatval($dat->ai_relation_score) : null;

                if($ai_score === null || $ai_score <= 0){
                    $source_post = new Wpil_Model_Post($dat->post_id, $dat->post_type);
                    $target_post = new Wpil_Model_Post($dat->target_id, $dat->target_type);
                    $ai_score = self::get_post_relationship_score($source_post, $target_post);

                    if($ai_score > 0){
                        $wpdb->update($linking_table, ['ai_relation_score' => $ai_score], ['ai_index' => $dat->ai_index]);
                    }
                }

                if($ai_score === null || $ai_score < $auto_insert_threshold || $ai_score > 0.9999){
                    $wpdb->update($linking_table, ['ignored' => 1], ['ai_index' => $dat->ai_index]);
                    continue;
                }
            }
            
            $source_post = new Wpil_Model_Post($dat->post_id, $dat->post_type);
            $target_post = new Wpil_Model_Post($dat->target_id, $dat->target_type);

            // if we're at the linking limits
            if( self::has_reached_ai_outbound_processing_limit($source_post) ||
                ($max_outbound > 0 && $source_post->getOutboundInternalLinks(true) >= $max_outbound) ||
                ($max_inbound > 0 && $target_post->getInboundInternalLinks(true) >= $max_inbound) ||
                ($orphan_only_targets && $target_post->getInboundInternalLinks(true) > 0)
            ){
                // marke the link as processed
                $wpdb->update($linking_table, ['ignored' => 1], ['ai_index' => $dat->ai_index]);
                // and skip inserting this suggestion
                continue;
            }

            $result = self::apply_linking_suggestion($dat, true);
            if(!empty($result['applied'])){
                $links_inserted++;
            }
        }

        if(!empty($links_inserted)){
            Wpil_Telemetry::log_event('linkwhisper_ai_link_inserted', array('count' => $links_inserted));
        }

        return false; // just assume that we're not done with processing
    }

    /**
     * Marks the inserted suggestion row and suppresses duplicate target suggestions for the same sentence.
     *
     * @param object $suggestion
     * @return void
     */
    private static function mark_suggestion_inserted_and_ignore_sentence_matches($suggestion){
        global $wpdb;
        $linking_table = $wpdb->prefix . 'wpil_ai_linking';

        if(empty($suggestion) || !is_object($suggestion) || empty($suggestion->ai_index)){
            return;
        }

        $sentence_id = !empty($suggestion->sentence_id) ? (string)$suggestion->sentence_id : '';
        if($sentence_id === '' && !empty($suggestion->sentence_text)){
            $sentence_id = md5((string)$suggestion->sentence_text);
        }
        $process_key = !empty($suggestion->process_key) ? (string)$suggestion->process_key : '';

        $wpdb->update($linking_table, ['inserted' => 1], ['ai_index' => (int)$suggestion->ai_index]);

        $sentence_text = !empty($suggestion->sentence_text) ? (string)$suggestion->sentence_text : '';

        // Independent rule 1: ignore all sibling suggestions for the same source sentence.
        if($sentence_id !== ''){
            $sql = "UPDATE {$linking_table}
                    SET ignored = 1
                    WHERE ai_index != %d
                        AND inserted < 1
                        AND ignored < 1
                        AND post_id = %d
                        AND post_type = %s
                        AND (
                            sentence_id = %s
                            OR ((sentence_id = '' OR sentence_id IS NULL) AND sentence_text = %s)
                        )";
            $params = [
                (int)$suggestion->ai_index,
                (int)$suggestion->post_id,
                (string)$suggestion->post_type,
                $sentence_id,
                $sentence_text,
            ];
            if($process_key !== ''){
                $sql .= " AND process_key = %s";
                $params[] = $process_key;
            }

            $wpdb->query($wpdb->prepare($sql, $params));
        }

        // Independent rule 2: ignore all sibling suggestions for the same source -> target pair.
        $sql = "UPDATE {$linking_table}
                SET ignored = 1
                WHERE ai_index != %d
                    AND inserted < 1
                    AND ignored < 1
                    AND post_id = %d
                    AND post_type = %s
                    AND target_id = %d
                    AND target_type = %s";
        $params = [
            (int)$suggestion->ai_index,
            (int)$suggestion->post_id,
            (string)$suggestion->post_type,
            (int)$suggestion->target_id,
            (string)$suggestion->target_type,
        ];
        if($process_key !== ''){
            $sql .= " AND process_key = %s";
            $params[] = $process_key;
        }

        $wpdb->query($wpdb->prepare($sql, $params));
    }


    /**
     * @param object $linking_data
     **/
    public static function build_target_link_from_suggestion($linking_data = array()){
        if(empty($linking_data)){
            return false;
        }

        // get the target URL
        $target = new Wpil_Model_Post($linking_data->target_id, $linking_data->target_type);

        // make sure the post exists
        if(!$target->check_if_post_exists() || empty($target->getViewLink())){
            return false;
        }

        // get the anchor
        preg_match_all('~<a\b[^>]*href\s*=\s*([\'"])wpil-link-proposal\1[^>]*>(.*?)</a>~is', $linking_data->sentence_with_anchor_text, $m);

        if(!isset($m[2]) || empty($m[2])){
            return false;
        }

        $anchor = $m[2][0];

        // create a clean sentence version
        $sentence = preg_replace('~<a\b[^>]*href\s*=\s*([\'"])wpil-link-proposal\1[^>]*>(.*?)</a>~is', '$2', $linking_data->sentence_with_anchor_text);
        
        // build the link
        $link = Wpil_Post::build_link_with_attrs($target->getViewLink(), $anchor, $sentence);

        // and return it
        return $link;
    }

    public static function ajax_get_review_links(){
        global $wpdb;
        $table = $wpdb->prefix . 'wpil_ai_linking';
        $report_links_table = $wpdb->prefix . 'wpil_report_links';

        Wpil_Base::verify_nonce('wizard-scanning-nonce');

        if (!current_user_can('manage_options')) {
            wp_send_json_error(array('message' => 'No permission'), 403);
        }

        $visible_limit = isset($_POST['visible_limit']) ? (int) wp_unslash($_POST['visible_limit']) : 5;
        if($visible_limit < 3){
            $visible_limit = 3;
        }elseif($visible_limit > 30){
            $visible_limit = 30;
        }

        $max = $visible_limit;
        $fetch_limit = max(50, ($visible_limit * 4));
        $sort_key = !empty($_POST['sort_key']) ? sanitize_key(wp_unslash($_POST['sort_key'])) : 'ai';
        $sort_dir = !empty($_POST['sort_dir']) ? strtolower(sanitize_text_field(wp_unslash($_POST['sort_dir']))) : 'desc';
        $sort_dir = ($sort_dir === 'asc') ? 'asc' : 'desc';
        $has_process_key = isset($_POST['process_key']);
        $process_key = $has_process_key ? sanitize_text_field(wp_unslash($_POST['process_key'])) : '';
        if(!$has_process_key && empty($process_key)){
            $process_key = get_option('wpil_ai_linking_process_key', '');
        }
        $fix_type = !empty($_POST['fix_type']) ? sanitize_key(wp_unslash($_POST['fix_type'])) : '';
        if($fix_type === 'orphaned_posts' && !self::is_orphan_fix_process_key($process_key)){
            $process_key = md5('orphan-post-search');
        }
        $orphan_only_targets = self::is_orphan_fix_process_key($process_key);
        $dashboard_fix_keys = self::get_dashboard_fix_process_keys();
        $is_dashboard_fix_process = (!empty($process_key) && in_array($process_key, $dashboard_fix_keys, true));
        $fix_special_options = $is_dashboard_fix_process ? Wpil_Settings::get_ai_fix_special_options($process_key) : array();
        $fix_filter_sql = '';
        $fix_filter_params = array();
        if(!empty($fix_special_options['link_to_category_pages'])){
            $fix_filter_sql .= " AND a.target_type = 'term'";
        }
        if(!empty($fix_special_options['link_from_category_pages'])){
            $fix_filter_sql .= " AND a.post_type = 'term'";
        }
        if(!empty($fix_special_options['select_post_types']) && !empty($fix_special_options['selected_post_types']) && is_array($fix_special_options['selected_post_types'])){
            $selected_types = array_values(array_unique(array_filter(array_map('sanitize_text_field', $fix_special_options['selected_post_types']))));
            if(!empty($selected_types)){
                $type_placeholders = implode(',', array_fill(0, count($selected_types), '%s'));
                $fix_filter_sql .= " AND (
                    a.target_type <> 'post'
                    OR EXISTS (
                        SELECT 1
                        FROM {$wpdb->posts} fix_posts
                        WHERE fix_posts.ID = a.target_id
                            AND fix_posts.post_type IN ({$type_placeholders})
                    )
                )";
                $fix_filter_params = array_merge($fix_filter_params, $selected_types);
            }
        }

        $hidden_link_ids = get_transient('wpil_hide_ai_link_id');
        if(!is_array($hidden_link_ids)){
            $hidden_link_ids = array();
        }
        $hidden_link_ids = array_values(array_unique(array_filter(array_map('intval', $hidden_link_ids))));

        $table_data = $wpdb->get_row("SELECT table_collation, SUBSTRING_INDEX(table_collation, '_', 1) AS character_set FROM information_schema.tables WHERE table_schema = '{$wpdb->dbname}' AND table_name = '{$wpdb->posts}'");
        $report_collation = (!empty($table_data) && !empty($table_data->table_collation)) ? $table_data->table_collation : 'utf8mb4_unicode_ci';
        $report_charset = (!empty($table_data) && !empty($table_data->character_set)) ? $table_data->character_set : 'utf8mb4';
        $target_type_compare_sql = "CONVERT(rl.target_type USING {$report_charset}) COLLATE {$report_collation} = CONVERT(a.target_type USING {$report_charset}) COLLATE {$report_collation}";
        $post_type_compare_sql = "CONVERT(rl.post_type USING {$report_charset}) COLLATE {$report_collation} = CONVERT(a.target_type USING {$report_charset}) COLLATE {$report_collation}";

        $allowed_sort_keys = array('ai', 'type', 'tax', 'inbound', 'outbound_internal', 'outbound_external', 'post_id', 'language');
        if(!in_array($sort_key, $allowed_sort_keys, true)){
            $sort_key = 'ai';
        }
        $source_filter_data = self::get_review_source_filter_sql('a', $report_links_table, $report_charset, $report_collation);

        $params = array();
        $where = "WHERE a.inserted = 0 AND a.ignored = 0";
        if($has_process_key && $process_key === ''){
            $where .= " AND 1 = 0";
        }
        if(!empty($process_key) && $process_key !== 'all'){
            $where .= " AND a.ai_relation_score < 1 AND a.process_key = %s";
            $params[] = $process_key;
        }
        if($orphan_only_targets){
            $where .= " AND NOT EXISTS (
                SELECT 1
                FROM {$report_links_table} rl
                WHERE rl.target_id = a.target_id
                    AND ((a.target_type = 'post' AND rl.target_type = 'post') OR (a.target_type = 'term' AND rl.target_type = 'term'))
            )";
        }
        if(!empty($fix_filter_sql)){
            $where .= $fix_filter_sql;
            $params = array_merge($params, $fix_filter_params);
        }
        if(!empty($hidden_link_ids)){
            $where .= " AND a.ai_index NOT IN (" . implode(',', array_fill(0, count($hidden_link_ids), '%d')) . ")";
            $params = array_merge($params, $hidden_link_ids);
        }
        if(!empty($source_filter_data['sql'])){
            $where .= $source_filter_data['sql'];
            $params = array_merge($params, $source_filter_data['params']);
        }

        $select_sql = "SELECT a.*";
        $order_sql = "ORDER BY a.ai_relation_score DESC, a.ai_index DESC";
        if(in_array($sort_key, array('inbound', 'outbound_internal', 'outbound_external'), true)){
            switch($sort_key){
                case 'inbound':
                    $sort_count_sql = "(SELECT COUNT(*) FROM {$report_links_table} rl WHERE rl.target_id = a.target_id AND {$target_type_compare_sql})";
                    break;
                case 'outbound_internal':
                    $sort_count_sql = "(SELECT COUNT(*) FROM {$report_links_table} rl WHERE rl.post_id = a.target_id AND {$post_type_compare_sql} AND rl.internal = 1)";
                    break;
                case 'outbound_external':
                default:
                    $sort_count_sql = "(SELECT COUNT(*) FROM {$report_links_table} rl WHERE rl.post_id = a.target_id AND {$post_type_compare_sql} AND rl.internal = 0)";
                    break;
            }

            $select_sql .= ", {$sort_count_sql} AS report_sort_count";
            $order_sql = "ORDER BY report_sort_count {$sort_dir}, a.ai_relation_score DESC, a.ai_index DESC";
        }

        $sql = "{$select_sql} FROM {$table} a {$where} {$order_sql} LIMIT %d";
        $params[] = $fetch_limit;
        $prepared = $wpdb->prepare($sql, $params);
        $rows = $wpdb->get_results($prepared, ARRAY_A);

        $items = array();
        if (!empty($rows)) {
            foreach ($rows as $row) {
                $target_title = '';
                $target_hint  = '';
                $target_data  = array(
                    'title' => '',
                    'type' => '',
                    'taxonomy' => '',
                    'categories' => '',
                    'tags' => '',
                    'inbound_internal' => 0,
                    'outbound_internal' => 0,
                    'outbound_external' => 0,
                    'post_id' => 0,
                    'language' => '',
                    'view_link' => '',
                    'ai_relatedness' => ''
                );
                $source_data = array(
                    'title' => '',
                    'type' => '',
                    'taxonomy' => '',
                    'categories' => '',
                    'tags' => '',
                    'inbound_internal' => 0,
                    'outbound_internal' => 0,
                    'outbound_external' => 0,
                    'post_id' => 0,
                    'language' => '',
                    'view_link' => '',
                    'ai_relatedness' => ''
                );

                $source_post = null;
                $source_view_link = '';
                $ai_relatedness = '';
                if(isset($row['ai_relation_score']) && is_numeric($row['ai_relation_score']) && !empty($row['ai_relation_score'])){
                    $ai_relatedness = (round((float)$row['ai_relation_score'], 4) * 100) . '%';
                }
                if(!empty($row['post_id']) && !empty($row['post_type'])){
                    $source_post = new Wpil_Model_Post((int)$row['post_id'], $row['post_type']);
                    if(!empty($source_post)){
                        $source_view_link = $source_post->getLinks()->view;
                        if(!empty($source_view_link)){
                            $source_view_link = Wpil_Link::filter_staging_to_live_domain($source_view_link);
                        }
                        $source_data = self::get_review_post_display_data($source_post, $ai_relatedness);
                    }
                }

                if (!empty($row['target_id']) && !empty($row['target_type'])) {
                    $target_post = new Wpil_Model_Post((int)$row['target_id'], $row['target_type']);
                    $target_title = $target_post->getTitle();
                    $target_state = self::get_live_target_state((int) $row['target_id'], (string) $row['target_type']);

                    if($orphan_only_targets && empty($target_state['orphan_eligible'])){
                        continue;
                    }
                    $target_data = self::get_review_post_display_data($target_post, $ai_relatedness);
                }

                $sentence_link = self::build_target_link_from_suggestion((object)$row);
                if(empty($sentence_link)){
                    self::ignore_linking_suggestions([(int)$row['ai_index']]);
                    continue;
                }

                $sentence_id = isset($row['sentence_id']) ? (string)$row['sentence_id'] : '';
                if($sentence_id === '' && !empty($row['sentence_text'])){
                    $sentence_id = md5((string)$row['sentence_text']);
                }

                $items[] = array(
                    'id' => (int)$row['ai_index'],
                    'post_id' => (int)$row['post_id'],
                    'post_type' => isset($row['post_type']) ? (string)$row['post_type'] : '',
                    'sentence_id' => $sentence_id,
                    'target_id' => (int)$row['target_id'],
                    'target_type' => isset($row['target_type']) ? (string)$row['target_type'] : '',
                    'post_title' => (!empty($source_post) ? $source_post->getTitle() : get_the_title((int)$row['post_id'])),
                    'source_view_link' => $source_view_link,
                    'target_title' => $target_title,
                    'target_hint'  => $target_hint,
                    'proposed_sentence_html' => $sentence_link,
                    'ai_relation_score' => isset($row['ai_relation_score']) ? floatval($row['ai_relation_score']) : null,
                    'source_data' => $source_data,
                    'target_data' => $target_data,
                );
            }
        }

        if(!empty($items)){
            $numeric_sort_keys = array(
                'ai' => true,
                'inbound' => true,
                'outbound_internal' => true,
                'outbound_external' => true,
                'post_id' => true
            );

            $extract_sort_value = function($item) use ($sort_key){
                $target_data = isset($item['target_data']) && is_array($item['target_data']) ? $item['target_data'] : array();
                switch($sort_key){
                    case 'ai':
                        return isset($item['ai_relation_score']) ? $item['ai_relation_score'] : null;
                    case 'type':
                        return isset($target_data['type']) ? $target_data['type'] : '';
                    case 'tax':
                        if(!empty($target_data['categories'])) return $target_data['categories'];
                        if(!empty($target_data['tags'])) return $target_data['tags'];
                        return isset($target_data['taxonomy']) ? $target_data['taxonomy'] : '';
                    case 'inbound':
                        return isset($target_data['inbound_internal']) ? $target_data['inbound_internal'] : null;
                    case 'outbound_internal':
                        return isset($target_data['outbound_internal']) ? $target_data['outbound_internal'] : null;
                    case 'outbound_external':
                        return isset($target_data['outbound_external']) ? $target_data['outbound_external'] : null;
                    case 'post_id':
                        return isset($target_data['post_id']) ? $target_data['post_id'] : null;
                    case 'language':
                        return isset($target_data['language']) ? $target_data['language'] : '';
                    default:
                        return isset($item['ai_relation_score']) ? $item['ai_relation_score'] : null;
                }
            };

            usort($items, function($a, $b) use ($extract_sort_value, $numeric_sort_keys, $sort_key, $sort_dir){
                $a_val = $extract_sort_value($a);
                $b_val = $extract_sort_value($b);
                $is_numeric = !empty($numeric_sort_keys[$sort_key]);

                if($is_numeric){
                    $a_missing = !is_numeric($a_val);
                    $b_missing = !is_numeric($b_val);
                    if($a_missing !== $b_missing){
                        return $a_missing ? 1 : -1;
                    }
                    if(!$a_missing){
                        $a_num = (float) $a_val;
                        $b_num = (float) $b_val;
                        if($a_num !== $b_num){
                            if($sort_dir === 'asc'){
                                return ($a_num < $b_num) ? -1 : 1;
                            }
                            return ($a_num > $b_num) ? -1 : 1;
                        }
                    }
                }else{
                    $a_str = trim(strtolower((string) $a_val));
                    $b_str = trim(strtolower((string) $b_val));
                    $a_missing = ($a_str === '');
                    $b_missing = ($b_str === '');
                    if($a_missing !== $b_missing){
                        return $a_missing ? 1 : -1;
                    }
                    if(!$a_missing && $a_str !== $b_str){
                        if($sort_dir === 'asc'){
                            return ($a_str < $b_str) ? -1 : 1;
                        }
                        return ($a_str > $b_str) ? -1 : 1;
                    }
                }

                $a_score = (isset($a['ai_relation_score']) && is_numeric($a['ai_relation_score'])) ? (float)$a['ai_relation_score'] : -INF;
                $b_score = (isset($b['ai_relation_score']) && is_numeric($b['ai_relation_score'])) ? (float)$b['ai_relation_score'] : -INF;
                if($a_score !== $b_score){
                    return ($a_score > $b_score) ? -1 : 1;
                }

                $a_id = isset($a['id']) ? (int)$a['id'] : 0;
                $b_id = isset($b['id']) ? (int)$b['id'] : 0;
                if($a_id === $b_id){
                    return 0;
                }
                return ($a_id > $b_id) ? -1 : 1;
            });
        }

        if(count($items) > $max){
            $items = array_slice($items, 0, $max);
        }

        if($has_process_key && $process_key === ''){
            $remaining = 0;
        }elseif(!empty($process_key) && $process_key !== 'all'){
            $remaining_where = "a.inserted = 0 AND a.ignored = 0 AND a.ai_relation_score < 1 AND a.process_key = %s";
            $remaining_params = array($process_key);
            if($orphan_only_targets){
                $remaining_where .= " AND NOT EXISTS (
                    SELECT 1
                    FROM {$report_links_table} rl
                    WHERE rl.target_id = a.target_id
                        AND ((a.target_type = 'post' AND rl.target_type = 'post') OR (a.target_type = 'term' AND rl.target_type = 'term'))
                )";
            }
            if(!empty($fix_filter_sql)){
                $remaining_where .= $fix_filter_sql;
                $remaining_params = array_merge($remaining_params, $fix_filter_params);
            }
            if(!empty($hidden_link_ids)){
                $remaining_where .= " AND a.ai_index NOT IN (" . implode(',', array_fill(0, count($hidden_link_ids), '%d')) . ")";
                $remaining_params = array_merge($remaining_params, $hidden_link_ids);
            }
            if(!empty($source_filter_data['sql'])){
                $remaining_where .= $source_filter_data['sql'];
                $remaining_params = array_merge($remaining_params, $source_filter_data['params']);
            }

            $remaining_sql = "SELECT COUNT(*) FROM {$table} a WHERE {$remaining_where}";
            $remaining = (int) $wpdb->get_var($wpdb->prepare($remaining_sql, $remaining_params));
        }else{
            $remaining_where = "a.inserted = 0 AND a.ignored = 0";
            $remaining_params = array();
            if(!empty($hidden_link_ids)){
                $remaining_where .= " AND a.ai_index NOT IN (" . implode(',', array_fill(0, count($hidden_link_ids), '%d')) . ")";
                $remaining_params = array_merge($remaining_params, $hidden_link_ids);
            }
            if(!empty($source_filter_data['sql'])){
                $remaining_where .= $source_filter_data['sql'];
                $remaining_params = array_merge($remaining_params, $source_filter_data['params']);
            }

            $remaining_sql = "SELECT COUNT(*) FROM {$table} a WHERE {$remaining_where}";
            $remaining = !empty($remaining_params) ? (int) $wpdb->get_var($wpdb->prepare($remaining_sql, $remaining_params)) : (int) $wpdb->get_var($remaining_sql);
        }

        wp_send_json_success(array(
            'items' => $items,
            'remaining' => $remaining,
            'process_key' => $process_key,
        ));
    }

    private static function sanitize_review_source_filter_date($date = ''){
        $date = sanitize_text_field((string) $date);
        if(empty($date) || !preg_match('/^\d{4}-\d{2}-\d{2}$/', $date)){
            return '';
        }

        $date_parts = explode('-', $date);
        if(3 !== count($date_parts)){
            return '';
        }

        $year = (int) $date_parts[0];
        $month = (int) $date_parts[1];
        $day = (int) $date_parts[2];

        return checkdate($month, $day, $year) ? sprintf('%04d-%02d-%02d', $year, $month, $day) : '';
    }

    private static function get_review_source_filter_sql($table_alias = 'a', $report_links_table = '', $report_charset = 'utf8mb4', $report_collation = 'utf8mb4_unicode_ci'){
        global $wpdb;

        $table_alias = preg_replace('/[^a-z0-9_]/i', '', (string) $table_alias);
        if(empty($table_alias)){
            $table_alias = 'a';
        }

        $report_links_table = !empty($report_links_table) ? $report_links_table : ($wpdb->prefix . 'wpil_report_links');
        $report_charset = preg_replace('/[^a-z0-9_]/i', '', (string) $report_charset);
        $report_collation = preg_replace('/[^a-z0-9_]/i', '', (string) $report_collation);
        if(empty($report_charset)){
            $report_charset = 'utf8mb4';
        }
        if(empty($report_collation)){
            $report_collation = 'utf8mb4_unicode_ci';
        }

        $source_date_after = isset($_POST['source_date_after']) ? self::sanitize_review_source_filter_date(wp_unslash($_POST['source_date_after'])) : '';
        $source_link_metric = isset($_POST['source_link_metric']) ? sanitize_key(wp_unslash($_POST['source_link_metric'])) : '';
        $source_link_compare = isset($_POST['source_link_compare']) ? sanitize_key(wp_unslash($_POST['source_link_compare'])) : '';
        $source_link_value = isset($_POST['source_link_value']) ? trim((string) wp_unslash($_POST['source_link_value'])) : '';

        if(!in_array($source_link_metric, array('inbound', 'outbound_internal'), true)){
            $source_link_metric = '';
        }

        if(!in_array($source_link_compare, array('gt', 'eq', 'lt'), true)){
            $source_link_compare = '';
        }

        if('' !== $source_link_value){
            if(ctype_digit($source_link_value)){
                $source_link_value = (int) $source_link_value;
            }else{
                $source_link_value = null;
            }
        }else{
            $source_link_value = null;
        }

        $sql = '';
        $params = array();

        if(!empty($source_date_after)){
            $sql .= " AND {$table_alias}.post_type = 'post'
                AND EXISTS (
                    SELECT 1
                    FROM {$wpdb->posts} review_source_posts
                    WHERE review_source_posts.ID = {$table_alias}.post_id
                        AND review_source_posts.post_date >= %s
                )";
            $params[] = $source_date_after . ' 00:00:00';
        }

        if(!empty($source_link_metric) && !empty($source_link_compare) && null !== $source_link_value){
            $operator = '=';
            if('gt' === $source_link_compare){
                $operator = '>';
            }elseif('lt' === $source_link_compare){
                $operator = '<';
            }

            $source_target_type_compare_sql = "CONVERT(source_rl.target_type USING {$report_charset}) COLLATE {$report_collation} = CONVERT({$table_alias}.post_type USING {$report_charset}) COLLATE {$report_collation}";
            $source_post_type_compare_sql = "CONVERT(source_rl.post_type USING {$report_charset}) COLLATE {$report_collation} = CONVERT({$table_alias}.post_type USING {$report_charset}) COLLATE {$report_collation}";
            if('outbound_internal' === $source_link_metric){
                $source_link_count_sql = "(SELECT COUNT(*) FROM {$report_links_table} source_rl WHERE source_rl.post_id = {$table_alias}.post_id AND {$source_post_type_compare_sql} AND source_rl.internal = 1)";
            }else{
                $source_link_count_sql = "(SELECT COUNT(*) FROM {$report_links_table} source_rl WHERE source_rl.target_id = {$table_alias}.post_id AND {$source_target_type_compare_sql} AND source_rl.internal = 1)";
            }

            $sql .= " AND ({$source_link_count_sql}) {$operator} %d";
            $params[] = $source_link_value;
        }

        return array(
            'sql' => $sql,
            'params' => $params,
        );
    }

    public static function ajax_get_review_link_count(){
        global $wpdb;
        $table = $wpdb->prefix . 'wpil_ai_linking';
        $report_links_table = $wpdb->prefix . 'wpil_report_links';

        Wpil_Base::verify_nonce('wizard-scanning-nonce');

        if (!current_user_can('manage_options')) {
            wp_send_json_error(array('message' => 'No permission'), 403);
        }

        $has_process_key = isset($_POST['process_key']);
        $process_key = $has_process_key ? sanitize_text_field(wp_unslash($_POST['process_key'])) : '';
        if(!$has_process_key && empty($process_key)){
            $process_key = get_option('wpil_ai_linking_process_key', '');
        }
        $fix_type = !empty($_POST['fix_type']) ? sanitize_key(wp_unslash($_POST['fix_type'])) : '';
        if($fix_type === 'orphaned_posts' && !self::is_orphan_fix_process_key($process_key)){
            $process_key = md5('orphan-post-search');
        }
        $orphan_only_targets = self::is_orphan_fix_process_key($process_key);
        $dashboard_fix_keys = self::get_dashboard_fix_process_keys();
        $is_dashboard_fix_process = (!empty($process_key) && in_array($process_key, $dashboard_fix_keys, true));
        $fix_special_options = $is_dashboard_fix_process ? Wpil_Settings::get_ai_fix_special_options($process_key) : array();
        $fix_filter_sql = '';
        $fix_filter_params = array();
        if(!empty($fix_special_options['link_to_category_pages'])){
            $fix_filter_sql .= " AND a.target_type = 'term'";
        }
        if(!empty($fix_special_options['link_from_category_pages'])){
            $fix_filter_sql .= " AND a.post_type = 'term'";
        }
        if(!empty($fix_special_options['select_post_types']) && !empty($fix_special_options['selected_post_types']) && is_array($fix_special_options['selected_post_types'])){
            $selected_types = array_values(array_unique(array_filter(array_map('sanitize_text_field', $fix_special_options['selected_post_types']))));
            if(!empty($selected_types)){
                $type_placeholders = implode(',', array_fill(0, count($selected_types), '%s'));
                $fix_filter_sql .= " AND (
                    a.target_type <> 'post'
                    OR EXISTS (
                        SELECT 1
                        FROM {$wpdb->posts} fix_posts
                        WHERE fix_posts.ID = a.target_id
                            AND fix_posts.post_type IN ({$type_placeholders})
                    )
                )";
                $fix_filter_params = array_merge($fix_filter_params, $selected_types);
            }
        }

        $hidden_link_ids = get_transient('wpil_hide_ai_link_id');
        if(!is_array($hidden_link_ids)){
            $hidden_link_ids = array();
        }
        $hidden_link_ids = array_values(array_unique(array_filter(array_map('intval', $hidden_link_ids))));

        if($has_process_key && $process_key === ''){
            $remaining = 0;
        }elseif(!empty($process_key) && $process_key !== 'all'){
            $remaining_where = "a.inserted = 0 AND a.ignored = 0 AND a.ai_relation_score < 1 AND a.process_key = %s";
            $remaining_params = array($process_key);
            if($orphan_only_targets){
                $remaining_where .= " AND NOT EXISTS (
                    SELECT 1
                    FROM {$report_links_table} rl
                    WHERE rl.target_id = a.target_id
                        AND ((a.target_type = 'post' AND rl.target_type = 'post') OR (a.target_type = 'term' AND rl.target_type = 'term'))
                )";
            }
            if(!empty($fix_filter_sql)){
                $remaining_where .= $fix_filter_sql;
                $remaining_params = array_merge($remaining_params, $fix_filter_params);
            }
            if(!empty($hidden_link_ids)){
                $remaining_where .= " AND a.ai_index NOT IN (" . implode(',', array_fill(0, count($hidden_link_ids), '%d')) . ")";
                $remaining_params = array_merge($remaining_params, $hidden_link_ids);
            }

            $remaining_sql = "SELECT COUNT(*) FROM {$table} a WHERE {$remaining_where}";
            $remaining = (int) $wpdb->get_var($wpdb->prepare($remaining_sql, $remaining_params));
        }else{
            $remaining_where = "a.inserted = 0 AND a.ignored = 0";
            $remaining_params = array();
            if(!empty($hidden_link_ids)){
                $remaining_where .= " AND a.ai_index NOT IN (" . implode(',', array_fill(0, count($hidden_link_ids), '%d')) . ")";
                $remaining_params = array_merge($remaining_params, $hidden_link_ids);
            }

            $remaining_sql = "SELECT COUNT(*) FROM {$table} a WHERE {$remaining_where}";
            $remaining = !empty($remaining_params) ? (int) $wpdb->get_var($wpdb->prepare($remaining_sql, $remaining_params)) : (int) $wpdb->get_var($remaining_sql);
        }
        wp_send_json_success(array(
            'remaining' => $remaining,
            'process_key' => $process_key,
        ));
    }

    public static function wpil_parse_anchor_html($html){
        $out = array(
            'href' => '',
            'text' => '',
        );

        if (empty($html)) {
            return $out;
        }

        // Extract href
        if (preg_match('/<a\b[^>]*\bhref\s*=\s*([\'"])(.*?)\1/i', $html, $m)) {
            $out['href'] = html_entity_decode($m[2], ENT_QUOTES, 'UTF-8');
        } elseif (preg_match('/<a\b[^>]*\bhref\s*=\s*([^\'"\s>]+)/i', $html, $m)) {
            $out['href'] = html_entity_decode($m[1], ENT_QUOTES, 'UTF-8');
        }

        // Extract inner text (strip nested tags if any)
        if (preg_match('/<a\b[^>]*>(.*?)<\/a>/is', $html, $m)) {
            $text = trim(wp_strip_all_tags($m[1]));
            $out['text'] = html_entity_decode($text, ENT_QUOTES, 'UTF-8');
        }

        return $out;
    }

    /**
     * Prepare WordPress post content for link analysis by stripping non-essential syntax.
     *
     * - Removes Gutenberg block comments (keeps block inner HTML/text)
     * - Removes shortcode wrappers while preserving enclosed content
     * - Removes HTML attributes from opening tags, except href on <a> tags
     *
     * @param string $content
     * @return string
     */
    public static function prepare_post_content_for_linking($content){
        if(!is_string($content) || $content === ''){
            return '';
        }

        // Remove Gutenberg block comments like:
        // <!-- wp:paragraph {"align":"center"} -->
        // <!-- /wp:paragraph -->
        $content = preg_replace('/<!--\s*\/?wp:[\s\S]*?-->/i', '', $content);

        // Remove shortcode tags while preserving wrapped content where present.
        if(function_exists('get_shortcode_regex')){
            $pattern = '/' . get_shortcode_regex() . '/s';
            $content = preg_replace_callback($pattern, function($matches){
                // Handle escaped shortcodes like [[tag]]
                if(isset($matches[1], $matches[6]) && $matches[1] === '[' && $matches[6] === ']'){
                    return substr($matches[0], 1, -1);
                }

                // Keep enclosed content for non-self-closing shortcodes.
                if(isset($matches[5]) && $matches[5] !== ''){
                    return $matches[5];
                }

                return '';
            }, $content);
        }else{
            // Fallback: remove shortcode-like tags.
            $content = preg_replace('/\[(\/?)[^\[\]]+?\]/', '', $content);
        }

        // Strip all attributes from opening HTML tags except href in anchor tags.
        $content = preg_replace_callback('/<(?!\/|!|\?)([a-zA-Z][a-zA-Z0-9:-]*)(\s+[^>]*?)?(\s*\/?)>/', function($matches){
            $tag_name = strtolower($matches[1]);
            $is_self_closing = (isset($matches[3]) && strpos($matches[3], '/') !== false);

            if($tag_name !== 'a'){
                return '<' . $matches[1] . ($is_self_closing ? ' /' : '') . '>';
            }

            $attrs = isset($matches[2]) ? $matches[2] : '';
            $href = '';
            if(preg_match('/\bhref\s*=\s*(?:"([^"]*)"|\'([^\']*)\'|([^\s"\'>]+))/i', $attrs, $href_match)){
                if(isset($href_match[1]) && $href_match[1] !== ''){
                    $href = $href_match[1];
                }elseif(isset($href_match[2]) && $href_match[2] !== ''){
                    $href = $href_match[2];
                }elseif(isset($href_match[3])){
                    $href = $href_match[3];
                }
            }

            if($href === ''){
                return '<' . $matches[1] . '>';
            }else{
                // for the time being, use a placeholder for the href... Just to keep the bot from being stupid
                $href = 'https://en.wikipedia.org/'; // because wiki !== 'obvious testing domain'
            }

            return '<' . $matches[1] . ' href="' . esc_attr($href) . '"' . ($is_self_closing ? ' /' : '') . '>';
        }, $content);

        return $content;
    }

    public static function ajax_set_review_link_decision(){
        global $wpdb;
        $table = $wpdb->prefix . 'wpil_ai_linking';

        Wpil_Base::verify_nonce('wizard-scanning-nonce');

        $link_id = isset($_POST['link_id']) ? sanitize_text_field($_POST['link_id']) : '';
        $decision = isset($_POST['decision']) ? sanitize_text_field($_POST['decision']) : '';
        $has_process_key = isset($_POST['process_key']);
        $process_key = $has_process_key ? sanitize_text_field(wp_unslash($_POST['process_key'])) : '';
        if(!$has_process_key && empty($process_key)){
            $process_key = get_option('wpil_ai_linking_process_key', '');
        }
        $fix_type = !empty($_POST['fix_type']) ? sanitize_key(wp_unslash($_POST['fix_type'])) : '';
        if($fix_type === 'orphaned_posts' && !self::is_orphan_fix_process_key($process_key)){
            $process_key = md5('orphan-post-search');
        }

        if(empty($link_id) || ($decision !== 'approve' && $decision !== 'reject')){
            wp_send_json_error(array('message' => 'Invalid payload'), 400);
        }

        $id_list = get_transient('wpil_hide_ai_link_id');
        if(empty($id_list)){
            $id_list = [];
        }

        $id_list[] = $link_id;

        set_transient('wpil_hide_ai_link_id', $id_list, MINUTE_IN_SECONDS); // set a flag so we can tell what ids we're ignoring!

        if($has_process_key && $process_key === ''){
            wp_send_json_error(array('message' => 'Process key required'), 400);
        }

        $suggestion = null;
        if(!empty($process_key) && 'all' !== $process_key){
            $suggestion = $wpdb->get_row($wpdb->prepare(
                "SELECT * FROM {$table} WHERE ai_index = %d AND process_key = %s LIMIT 1",
                (int) $link_id,
                $process_key
            ));
            if(empty($suggestion)){
                wp_send_json_error(array('message' => 'Suggestion not found for process'), 404);
            }
        }else{
            $suggestion = $wpdb->get_row($wpdb->prepare(
                "SELECT * FROM {$table} WHERE ai_index = %d LIMIT 1",
                (int) $link_id
            ));
            if(empty($suggestion)){
                wp_send_json_error(array('message' => 'Suggestion not found'), 404);
            }
        }

        if($decision === 'approve'){
            $result = self::create_links_from_linking_suggestions([$link_id]);
            wp_send_json_success(array(
                'link_id' => $link_id,
                'decision' => $decision,
                'process_key' => $process_key,
                'applied' => !empty($result['applied']),
                'stale' => !empty($result['stale']),
                'message' => !empty($result['message']) ? $result['message'] : '',
                'reason' => !empty($result['reason']) ? $result['reason'] : '',
            ));
        }else{
            self::ignore_linking_suggestions([$link_id]);
            wp_send_json_success(array(
                'link_id' => $link_id,
                'decision' => $decision,
                'process_key' => $process_key,
                'applied' => true,
                'stale' => false,
                'message' => 'Suggestion rejected.',
                'reason' => 'rejected',
            ));
        }
    }
}