diff --git a/application/controllers/Adif.php b/application/controllers/Adif.php index b1bb25194..00cc48a6c 100644 --- a/application/controllers/Adif.php +++ b/application/controllers/Adif.php @@ -85,17 +85,46 @@ class adif extends CI_Controller { } - // Export all QSO Data in ASC Order of Date. + // Export all QSO Data in ASC Order of Date - use chunks to avoid memory exhaustion public function exportall() { - // Set memory limit to unlimited to allow heavy usage $this->require_tab_access('export'); + ini_set('memory_limit', '-1'); + set_time_limit(300); $this->load->model('adif_data'); + $this->load->library('AdifHelper'); - $data['qsos'] = $this->adif_data->export_all(null, $this->input->post('from', true), $this->input->post('to', true)); + $from = $this->input->post('from', true); + $to = $this->input->post('to', true); - $this->load->view('adif/data/exportall', $data); + $filename = $this->session->userdata('user_callsign').'-'.date('Ymd-Hi').'.adi'; + header('Content-Type: text/plain; charset=utf-8'); + header('Content-Disposition: attachment; filename="'.$filename.'"'); + + // Output ADIF header // No chance to use exportall-view any longer, because of chunking logic + echo $this->adifhelper->getAdifHeader($this->config->item('app_name'),$this->optionslib->get_option('version')); + + // Stream QSOs in 5K chunks + $offset = 0; + $chunk_size = 5000; + + do { + $qsos = $this->adif_data->export_all_chunked(null, $from, $to, false, null, $offset, $chunk_size); + + if ($qsos->num_rows() > 0) { + foreach ($qsos->result() as $qso) { + echo $this->adifhelper->getAdifLine($qso); + } + + // Free memory + $qsos->free_result(); + } + + $offset += $chunk_size; + } while ($qsos->num_rows() > 0); + + exit; } @@ -143,10 +172,15 @@ class adif extends CI_Controller { // Set memory limit to unlimited to allow heavy usage ini_set('memory_limit', '-1'); + set_time_limit(300); $this->load->model('adif_data'); + $this->load->library('AdifHelper'); + // Get parameters $station_id = $this->security->xss_clean($this->input->post('station_profile')); + $from = $this->input->post('from'); + $to = $this->input->post('to'); // Used for exporting QSOs not previously exported to LoTW if ($this->input->post('exportLotw') == 1) { @@ -161,16 +195,45 @@ class adif extends CI_Controller { $onlyop=null; } - $data['qsos'] = $this->adif_data->export_custom($this->input->post('from'), $this->input->post('to'), $station_id, $exportLotw, $onlyop); + // Set headers for direct download + $filename = $this->session->userdata('user_callsign').'-'.date('Ymd-Hi').'.adi'; + header('Content-Type: text/plain; charset=utf-8'); + header('Content-Disposition: attachment; filename="'.$filename.'"'); - $this->load->view('adif/data/exportall', $data); + echo $this->adifhelper->getAdifHeader($this->config->item('app_name'),$this->optionslib->get_option('version')); + // Collect QSO IDs for LoTW marking (since we can't access all at once) + $qso_ids_for_lotw = []; - if ((clubaccess_check(9)) && ($this->input->post('markLotw') == 1)) { // Only allow marking if clubofficer or regular user! - foreach ($data['qsos']->result() as $qso) { - $this->adif_data->mark_lotw_sent($qso->COL_PRIMARY_KEY); + // Stream QSOs in 5K chunks + $offset = 0; + $chunk_size = 5000; + + do { + $qsos = $this->adif_data->export_custom_chunked($from, $to, $station_id, $exportLotw, $onlyop, $offset, $chunk_size); + + if ($qsos && $qsos->num_rows() > 0) { + foreach ($qsos->result() as $qso) { + echo $this->adifhelper->getAdifLine($qso); + // Collect IDs for LoTW marking + $qso_ids_for_lotw[] = $qso->COL_PRIMARY_KEY; + } + // Free memory + $qsos->free_result(); + } + + $offset += $chunk_size; + } while ($qsos && $qsos->num_rows() > 0); + + // Handle LoTW marking after export + if ((clubaccess_check(9)) && ($this->input->post('markLotw') == 1) && !empty($qso_ids_for_lotw)) { + foreach ($qso_ids_for_lotw as $qso_id) { + $this->adif_data->mark_lotw_sent($qso_id); } } + + // Stop execution + exit; } public function mark_lotw() { diff --git a/application/controllers/Api.php b/application/controllers/Api.php index 035e6fcd8..c07cb6e01 100644 --- a/application/controllers/Api.php +++ b/application/controllers/Api.php @@ -440,33 +440,64 @@ class API extends CI_Controller { //load adif data module $this->load->model('adif_data'); + $this->load->library('AdifHelper'); - //get qso data - $data['qsos'] = $this->adif_data->export_past_id($station_id, $fetchfromid, $limit); + // Initialize tracking variables + $total_fetched = 0; + $all_qso_ids = []; + $lastfetchedid = $fetchfromid; - //set internalonly attribute for adif creation - $data['internalrender'] = true; + // Process in chunks to avoid memory issues + $chunk_size = 5000; + $remaining_limit = $limit; + $offset = 0; - //if no new QSOs are ready, return that - $qso_count = count($data['qsos']->result()); - if($qso_count <= 0) { + // Start building ADIF content + $adif_content = $this->adifhelper->getAdifHeader($this->config->item('app_name'),$this->optionslib->get_option('version')); + + do { + // Calculate chunk size for this iteration + $current_chunk_size = min($chunk_size, $remaining_limit); + + // Fetch chunk + $qsos = $this->adif_data->export_past_id_chunked($station_id, $fetchfromid, $current_chunk_size, null, $offset, $current_chunk_size); + + if ($qsos && $qsos->num_rows() > 0) { + // Process chunk + foreach ($qsos->result() as $row) { + // Build ADIF content directly + $adif_content .= $this->adifhelper->getAdifLine($row); + + // Track data for response + $all_qso_ids[] = $row->COL_PRIMARY_KEY; + $lastfetchedid = max($lastfetchedid, $row->COL_PRIMARY_KEY); + $total_fetched++; + } + + // Free memory + $qsos->free_result(); + + // Update tracking + $remaining_limit -= $qsos->num_rows(); + $offset += $qsos->num_rows(); + + // Stop if we've hit the requested limit + if ($total_fetched >= $limit) { + break; + } + } + + // Continue if we got a full chunk and haven't hit the limit + } while ($qsos && $qsos->num_rows() > 0 && $total_fetched < $limit); + + // Return response (same format as original) + if($total_fetched <= 0) { http_response_code(200); echo json_encode(['status' => 'successfull', 'message' => 'No new QSOs available.', 'lastfetchedid' => $fetchfromid, 'exported_qsos' => 0, 'adif' => null]); - return; + } else { + http_response_code(200); + echo json_encode(['status' => 'successfull', 'message' => 'Export successfull', 'lastfetchedid' => $lastfetchedid, 'exported_qsos' => $total_fetched, 'adif' => $adif_content]); } - - //convert data to ADIF - $adif_content = $this->load->view('adif/data/exportall', $data, TRUE); - - //get new goalpost - $lastfetchedid = 0; - foreach ($data['qsos']->result() as $row) { - $lastfetchedid = max($lastfetchedid, $row->COL_PRIMARY_KEY); - } - - //return API result - http_response_code(200); - echo json_encode(['status' => 'successfull', 'message' => 'Export successfull', 'lastfetchedid' => $lastfetchedid, 'exported_qsos' => $qso_count, 'adif' => $adif_content]); } diff --git a/application/controllers/Backup.php b/application/controllers/Backup.php index 5c65e2028..945892e2e 100644 --- a/application/controllers/Backup.php +++ b/application/controllers/Backup.php @@ -29,31 +29,39 @@ class Backup extends CI_Controller { $clean_key = $this->security->xss_clean($key); $this->load->helper('file'); + $this->load->library('AdifHelper'); // Set memory limit to unlimited to allow heavy usage ini_set('memory_limit', '-1'); $this->load->model('adif_data'); + $filename = 'backup/logbook'. date('_Y_m_d_H_i_s') .'.adi'; - $data['qsos'] = $this->adif_data->export_all($clean_key); + header('Content-Type: text/plain; charset=utf-8'); + header('Content-Disposition: attachment; filename="'.$filename.'"'); - $data['filename'] = 'backup/logbook'. date('_Y_m_d_H_i_s') .'.adi'; - - if ( ! write_file($data['filename'], $this->load->view('backup/exportall', $data, true))) - { - $data['status'] = false; - } - else - { - $data['status'] = true; - } + // Output ADIF header // No chance to use exportall-view any longer, because of chunking logic + echo $this->adifhelper->getAdifHeader($this->config->item('app_name'),$this->optionslib->get_option('version')); - $data['page_title'] = __("ADIF - Backup"); - + // Stream QSOs in 5K chunks + $offset = 0; + $chunk_size = 5000; - $this->load->view('interface_assets/header', $data); - $this->load->view('backup/adif_view'); - $this->load->view('interface_assets/footer'); + do { + $qsos = $this->adif_data->export_all_chunked($clean_key, null, null, false, null, $offset, $chunk_size); + if ($qsos->num_rows() > 0) { + foreach ($qsos->result() as $qso) { + echo $this->adifhelper->getAdifLine($qso); + } + + // Free memory + $qsos->free_result(); + } + + $offset += $chunk_size; + } while ($qsos->num_rows() > 0); + + exit; } /* Export the notes to XML */ diff --git a/application/libraries/AdifHelper.php b/application/libraries/AdifHelper.php index 40adf9895..c317fd210 100644 --- a/application/libraries/AdifHelper.php +++ b/application/libraries/AdifHelper.php @@ -262,6 +262,15 @@ class AdifHelper { return $line; } + function getAdifHeader($app_name,$version) { + $adif_header = "Wavelog ADIF export\n"; + $adif_header .= "3.1.6\n"; + $adif_header .= "".$app_name."\r\n"; + $adif_header .= "".$version."\r\n"; + $adif_header .= "\n\n"; + return $adif_header; + } + function getAdifFieldLine($adifcolumn, $dbvalue) { if ($dbvalue !== "" && $dbvalue !== null && $dbvalue !== 0) { return "<" . $adifcolumn . ":" . mb_strlen($dbvalue, "UTF-8") . ">" . $dbvalue . "\r\n"; diff --git a/application/models/Adif_data.php b/application/models/Adif_data.php index e34845546..9dda46fce 100644 --- a/application/models/Adif_data.php +++ b/application/models/Adif_data.php @@ -2,6 +2,48 @@ class adif_data extends CI_Model { + function export_all_chunked($api_key = null, $from = null, $to = null, $exportLotw = false, $onlyop = null, $offset = 0, $limit = 5000) { + $this->load->model('logbooks_model'); + if ($api_key != null) { + $this->load->model('api_model'); + if (strpos($this->api_model->access($api_key), 'r') !== false) { + $this->api_model->update_last_used($api_key); + $user_id = $this->api_model->key_userid($api_key); + $logbooks_locations_array = $this->list_station_locations($user_id); + } + } else { + $this->load->model('stations'); + $logbooks_locations_array = $this->list_station_locations($this->session->userdata('user_id')); + } + + $this->db->select($this->config->item('table_name').'.*, station_profile.*, dxcc_entities.name as station_country'); + $this->db->order_by("COL_TIME_ON", "ASC"); + $this->db->join('station_profile', 'station_profile.station_id = '.$this->config->item('table_name').'.station_id'); + if ($from) { + $this->db->where("date(".$this->config->item('table_name').".COL_TIME_ON) >= ", $from); + } + if ($to) { + $this->db->where("date(".$this->config->item('table_name').".COL_TIME_ON) <= ",$to); + } + if ($onlyop) { + $this->db->where("upper(".$this->config->item('table_name').".col_operator)",$onlyop); + } + if ($exportLotw) { + $this->db->group_start(); + $this->db->where($this->config->item('table_name').".COL_LOTW_QSL_SENT != 'Y'"); + $this->db->or_where($this->config->item('table_name').".COL_LOTW_QSL_SENT", NULL); + $this->db->group_end(); + } + $this->db->where_in('station_profile.station_id', $logbooks_locations_array); + $this->db->join('dxcc_entities', 'station_profile.station_dxcc = dxcc_entities.adif'); + + // Add chunking + $this->db->limit($limit, $offset); + + $query = $this->db->get($this->config->item('table_name')); + return $query; + } + function export_all($api_key = null,$from = null, $to = null, $exportLotw = false, $onlyop = null) { $this->load->model('logbooks_model'); if ($api_key != null) { @@ -125,6 +167,52 @@ class adif_data extends CI_Model { return $this->db->get(); } + function export_custom_chunked($from, $to, $station_id, $exportLotw = false, $onlyop = null, $offset = 0, $limit = 5000) { + // Copy export_custom logic but add chunking for station_id > 0 + $this->load->model('Stations'); + if ($station_id == 0) { + // Use existing chunked export_all for all stations + return $this->export_all_chunked(null, $from, $to, $exportLotw, $onlyop, $offset, $limit); + } + + // Check station access + if (!$this->Stations->check_station_is_accessible($station_id)) { + return; + } + + // Build query identical to export_custom but add LIMIT/OFFSET + $this->db->select(''.$this->config->item('table_name').'.*, station_profile.*, dxcc_entities.name as station_country'); + $this->db->from($this->config->item('table_name')); + $this->db->where($this->config->item('table_name').'.station_id', $station_id); + + // Apply same filters as export_custom + if ($from) { + $this->db->where("date(".$this->config->item('table_name').".COL_TIME_ON) >= ", $from); + } + if ($to) { + $this->db->where("date(".$this->config->item('table_name').".COL_TIME_ON) <= ",$to); + } + if ($onlyop) { + $this->db->where("upper(".$this->config->item('table_name').".col_operator)",$onlyop); + } + if ($exportLotw) { + $this->db->group_start(); + $this->db->where($this->config->item('table_name').".COL_LOTW_QSL_SENT != 'Y'"); + $this->db->or_where($this->config->item('table_name').".COL_LOTW_QSL_SENT", NULL); + $this->db->group_end(); + } + + $this->db->order_by($this->config->item('table_name').".COL_TIME_ON", "ASC"); + $this->db->join('station_profile', 'station_profile.station_id = '.$this->config->item('table_name').'.station_id'); + $this->db->join('dxcc_entities', 'station_profile.station_dxcc = dxcc_entities.adif', 'left outer'); + + // Add chunking + $this->db->limit($limit, $offset); + + return $this->db->get(); + } + + function export_custom($from, $to, $station_id, $exportLotw = false, $onlyop = null) { // be sure that station belongs to user $this->load->model('Stations'); @@ -165,25 +253,24 @@ class adif_data extends CI_Model { } } - function export_past_id($station_id, $fetchfromid, $limit, $onlyop = null) { - //create query + function export_past_id_chunked($station_id, $fetchfromid, $limit, $onlyop = null, $offset = 0, $chunk_size = 5000) { + // Copy export_past_id logic but add chunking support $this->db->select(''.$this->config->item('table_name').'.*, station_profile.*, dxcc_entities.name as station_country'); $this->db->from($this->config->item('table_name')); $this->db->where($this->config->item('table_name').'.station_id', $station_id); - $this->db->where($this->config->item('table_name').".COL_PRIMARY_KEY > " , $fetchfromid); //only get values past the fetchfromid value - $this->db->order_by($this->config->item('table_name').".COL_TIME_ON", "ASC"); - $this->db->join('station_profile', 'station_profile.station_id = '.$this->config->item('table_name').'.station_id'); - $this->db->join('dxcc_entities', 'station_profile.station_dxcc = dxcc_entities.adif', 'left outer'); + $this->db->where($this->config->item('table_name').".COL_PRIMARY_KEY > ", $fetchfromid); + if ($onlyop) { $this->db->where("upper(".$this->config->item('table_name').".col_operator)",$onlyop); } + + // Add chunking + $this->db->limit($chunk_size, $offset); + $this->db->order_by("COL_PRIMARY_KEY", "ASC"); + $this->db->join('station_profile', 'station_profile.station_id = '.$this->config->item('table_name').'.station_id'); + $this->db->join('dxcc_entities', 'station_profile.station_dxcc = dxcc_entities.adif', 'left outer'); - if ($limit > -1) { - $this->db->limit($limit); - } - - //return result return $this->db->get(); } diff --git a/application/views/adif/data/exportall.php b/application/views/adif/data/exportall.php index 9f2b6b603..035eeb298 100644 --- a/application/views/adif/data/exportall.php +++ b/application/views/adif/data/exportall.php @@ -5,19 +5,14 @@ header('Content-Type: text/plain; charset=utf-8'); header('Content-Disposition: attachment; filename="'.$this->session->userdata('user_callsign').'-'.date('Ymd-Hi').'.adi"'); } -?> -Wavelog ADIF export -3.1.6 -config->item('app_name')); ?>>config->item('app_name')."\r\n"; ?> -optionslib->get_option('version')); ?>>optionslib->get_option('version')."\r\n"; ?> - - -load->is_loaded('AdifHelper')) { $CI->load->library('AdifHelper'); } +echo $CI->adifhelper->getAdifHeader($CI->config->item('app_name'),$CI->optionslib->get_option('version')); + foreach ($qsos->result() as $qso) { echo $CI->adifhelper->getAdifLine($qso); } +?>