diff --git a/application/controllers/Api.php b/application/controllers/Api.php index 035e6fcd8..6995c5353 100644 --- a/application/controllers/Api.php +++ b/application/controllers/Api.php @@ -440,33 +440,68 @@ class API extends CI_Controller { //load adif data module $this->load->model('adif_data'); + $this->load->library('AdifHelper'); - //get qso data - $data['qsos'] = $this->adif_data->export_past_id($station_id, $fetchfromid, $limit); + // Initialize tracking variables + $total_fetched = 0; + $all_qso_ids = []; + $lastfetchedid = $fetchfromid; - //set internalonly attribute for adif creation - $data['internalrender'] = true; + // Process in chunks to avoid memory issues + $chunk_size = 5000; + $remaining_limit = $limit; + $offset = 0; - //if no new QSOs are ready, return that - $qso_count = count($data['qsos']->result()); - if($qso_count <= 0) { + // Start building ADIF content + $adif_content = "Wavelog ADIF export\n"; + $adif_content .= "3.1.6\n"; + $adif_content .= "config->item('app_name')).">".$this->config->item('app_name')."\r\n"; + $adif_content .= "optionslib->get_option('version')).">".$this->optionslib->get_option('version')."\r\n"; + $adif_content .= "\n\n"; + + do { + // Calculate chunk size for this iteration + $current_chunk_size = min($chunk_size, $remaining_limit); + + // Fetch chunk + $qsos = $this->adif_data->export_past_id_chunked($station_id, $fetchfromid, $current_chunk_size, null, $offset, $current_chunk_size); + + if ($qsos && $qsos->num_rows() > 0) { + // Process chunk + foreach ($qsos->result() as $row) { + // Build ADIF content directly + $adif_content .= $this->adifhelper->getAdifLine($row); + + // Track data for response + $all_qso_ids[] = $row->COL_PRIMARY_KEY; + $lastfetchedid = max($lastfetchedid, $row->COL_PRIMARY_KEY); + $total_fetched++; + } + + // Free memory + $qsos->free_result(); + + // Update tracking + $remaining_limit -= $qsos->num_rows(); + $offset += $qsos->num_rows(); + + // Stop if we've hit the requested limit + if ($total_fetched >= $limit) { + break; + } + } + + // Continue if we got a full chunk and haven't hit the limit + } while ($qsos && $qsos->num_rows() > 0 && $total_fetched < $limit); + + // Return response (same format as original) + if($total_fetched <= 0) { http_response_code(200); echo json_encode(['status' => 'successfull', 'message' => 'No new QSOs available.', 'lastfetchedid' => $fetchfromid, 'exported_qsos' => 0, 'adif' => null]); - return; + } else { + http_response_code(200); + echo json_encode(['status' => 'successfull', 'message' => 'Export successfull', 'lastfetchedid' => $lastfetchedid, 'exported_qsos' => $total_fetched, 'adif' => $adif_content]); } - - //convert data to ADIF - $adif_content = $this->load->view('adif/data/exportall', $data, TRUE); - - //get new goalpost - $lastfetchedid = 0; - foreach ($data['qsos']->result() as $row) { - $lastfetchedid = max($lastfetchedid, $row->COL_PRIMARY_KEY); - } - - //return API result - http_response_code(200); - echo json_encode(['status' => 'successfull', 'message' => 'Export successfull', 'lastfetchedid' => $lastfetchedid, 'exported_qsos' => $qso_count, 'adif' => $adif_content]); } diff --git a/application/models/Adif_data.php b/application/models/Adif_data.php index e54ef1a16..98828a557 100644 --- a/application/models/Adif_data.php +++ b/application/models/Adif_data.php @@ -167,6 +167,52 @@ class adif_data extends CI_Model { return $this->db->get(); } + function export_custom_chunked($from, $to, $station_id, $exportLotw = false, $onlyop = null, $offset = 0, $limit = 5000) { + // Copy export_custom logic but add chunking for station_id > 0 + $this->load->model('Stations'); + if ($station_id == 0) { + // Use existing chunked export_all for all stations + return $this->export_all_chunked(null, $from, $to, $exportLotw, $onlyop, $offset, $limit); + } + + // Check station access + if (!$this->Stations->check_station_is_accessible($station_id)) { + return; + } + + // Build query identical to export_custom but add LIMIT/OFFSET + $this->db->select(''.$this->config->item('table_name').'.*, station_profile.*, dxcc_entities.name as station_country'); + $this->db->from($this->config->item('table_name')); + $this->db->where($this->config->item('table_name').'.station_id', $station_id); + + // Apply same filters as export_custom + if ($from) { + $this->db->where("date(".$this->config->item('table_name').".COL_TIME_ON) >= ", $from); + } + if ($to) { + $this->db->where("date(".$this->config->item('table_name').".COL_TIME_ON) <= ",$to); + } + if ($onlyop) { + $this->db->where("upper(".$this->config->item('table_name').".col_operator)",$onlyop); + } + if ($exportLotw) { + $this->db->group_start(); + $this->db->where($this->config->item('table_name').".COL_LOTW_QSL_SENT != 'Y'"); + $this->db->or_where($this->config->item('table_name').".COL_LOTW_QSL_SENT", NULL); + $this->db->group_end(); + } + + $this->db->order_by($this->config->item('table_name').".COL_TIME_ON", "ASC"); + $this->db->join('station_profile', 'station_profile.station_id = '.$this->config->item('table_name').'.station_id'); + $this->db->join('dxcc_entities', 'station_profile.station_dxcc = dxcc_entities.adif', 'left outer'); + + // Add chunking + $this->db->limit($limit, $offset); + + return $this->db->get(); + } + + function export_custom($from, $to, $station_id, $exportLotw = false, $onlyop = null) { // be sure that station belongs to user $this->load->model('Stations'); @@ -207,25 +253,25 @@ class adif_data extends CI_Model { } } - function export_past_id($station_id, $fetchfromid, $limit, $onlyop = null) { - //create query + function export_past_id_chunked($station_id, $fetchfromid, $limit, $onlyop = null, $offset = 0, $chunk_size = 5000) { + // Copy export_past_id logic but add chunking support $this->db->select(''.$this->config->item('table_name').'.*, station_profile.*, dxcc_entities.name as station_country'); $this->db->from($this->config->item('table_name')); $this->db->where($this->config->item('table_name').'.station_id', $station_id); - $this->db->where($this->config->item('table_name').".COL_PRIMARY_KEY > " , $fetchfromid); //only get values past the fetchfromid value - $this->db->order_by($this->config->item('table_name').".COL_TIME_ON", "ASC"); - $this->db->join('station_profile', 'station_profile.station_id = '.$this->config->item('table_name').'.station_id'); - $this->db->join('dxcc_entities', 'station_profile.station_dxcc = dxcc_entities.adif', 'left outer'); + $this->db->where($this->config->item('table_name').".COL_PRIMARY_KEY > ", $fetchfromid); + if ($onlyop) { $this->db->where("upper(".$this->config->item('table_name').".col_operator)",$onlyop); } + + // Add chunking + $this->db->limit($chunk_size, $offset); + + $this->db->order_by($this->config->item('table_name').".COL_TIME_ON", "ASC"); $this->db->order_by("COL_PRIMARY_KEY", "ASC"); + $this->db->join('station_profile', 'station_profile.station_id = '.$this->config->item('table_name').'.station_id'); + $this->db->join('dxcc_entities', 'station_profile.station_dxcc = dxcc_entities.adif', 'left outer'); - if ($limit > -1) { - $this->db->limit($limit); - } - - //return result return $this->db->get(); } @@ -280,53 +326,6 @@ class adif_data extends CI_Model { return $this->db->get(); } - - function export_custom_chunked($from, $to, $station_id, $exportLotw = false, $onlyop = null, $offset = 0, $limit = 5000) { - // Copy export_custom logic but add chunking for station_id > 0 - $this->load->model('Stations'); - if ($station_id == 0) { - // Use existing chunked export_all for all stations - return $this->export_all_chunked(null, $from, $to, $exportLotw, $onlyop, $offset, $limit); - } - - // Check station access - if (!$this->Stations->check_station_is_accessible($station_id)) { - return; - } - - // Build query identical to export_custom but add LIMIT/OFFSET - $this->db->select(''.$this->config->item('table_name').'.*, station_profile.*, dxcc_entities.name as station_country'); - $this->db->from($this->config->item('table_name')); - $this->db->where($this->config->item('table_name').'.station_id', $station_id); - - // Apply same filters as export_custom - if ($from) { - $this->db->where("date(".$this->config->item('table_name').".COL_TIME_ON) >= ", $from); - } - if ($to) { - $this->db->where("date(".$this->config->item('table_name').".COL_TIME_ON) <= ",$to); - } - if ($onlyop) { - $this->db->where("upper(".$this->config->item('table_name').".col_operator)",$onlyop); - } - if ($exportLotw) { - $this->db->group_start(); - $this->db->where($this->config->item('table_name').".COL_LOTW_QSL_SENT != 'Y'"); - $this->db->or_where($this->config->item('table_name').".COL_LOTW_QSL_SENT", NULL); - $this->db->group_end(); - } - - $this->db->order_by($this->config->item('table_name').".COL_TIME_ON", "ASC"); - $this->db->join('station_profile', 'station_profile.station_id = '.$this->config->item('table_name').'.station_id'); - $this->db->join('dxcc_entities', 'station_profile.station_dxcc = dxcc_entities.adif', 'left outer'); - - // Add chunking - $this->db->limit($limit, $offset); - - return $this->db->get(); - } - - } ?>