Public_Tools/wavelog
2
0
Fork 0
mirror of https://github.com/wavelog/wavelog.git synced 2026-03-22 10:24:14 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #2642 from int2001/exp_chunked

Browse Source 
Exp chunked
This commit is contained in:
Joerg (DJ7NT)
2025-12-05 17:49:08 +01:00
committed by GitHub
parent 3694c95b60 d70c21f277
commit c0415bfb04
6 changed files with 258 additions and 65 deletions
Download Patch File Download Diff File Expand all files Collapse all files

81
application/controllers/Adif.php
View File

@@ -85,17 +85,46 @@ class adif extends CI_Controller {
}
// Export all QSO Data in ASC Order of Date.
// Export all QSO Data in ASC Order of Date - use chunks to avoid memory exhaustion
public function exportall() {
// Set memory limit to unlimited to allow heavy usage
$this->require_tab_access('export');
ini_set('memory_limit', '-1');
set_time_limit(300);
$this->load->model('adif_data');
$this->load->library('AdifHelper');
$data['qsos'] = $this->adif_data->export_all(null, $this->input->post('from', true), $this->input->post('to', true));
$from = $this->input->post('from', true);
$to = $this->input->post('to', true);
$this->load->view('adif/data/exportall', $data);
$filename = $this->session->userdata('user_callsign').'-'.date('Ymd-Hi').'.adi';
header('Content-Type: text/plain; charset=utf-8');
header('Content-Disposition: attachment; filename="'.$filename.'"');
// Output ADIF header // No chance to use exportall-view any longer, because of chunking logic
echo $this->adifhelper->getAdifHeader($this->config->item('app_name'),$this->optionslib->get_option('version'));
// Stream QSOs in 5K chunks
$offset = 0;
$chunk_size = 5000;
do {
$qsos = $this->adif_data->export_all_chunked(null, $from, $to, false, null, $offset, $chunk_size);
if ($qsos->num_rows() > 0) {
foreach ($qsos->result() as $qso) {
echo $this->adifhelper->getAdifLine($qso);
}
// Free memory
$qsos->free_result();
}
$offset += $chunk_size;
} while ($qsos->num_rows() > 0);
exit;
}
@@ -143,10 +172,15 @@ class adif extends CI_Controller {
// Set memory limit to unlimited to allow heavy usage
ini_set('memory_limit', '-1');
set_time_limit(300);
$this->load->model('adif_data');
$this->load->library('AdifHelper');
// Get parameters
$station_id = $this->security->xss_clean($this->input->post('station_profile'));
$from = $this->input->post('from');
$to = $this->input->post('to');
// Used for exporting QSOs not previously exported to LoTW
if ($this->input->post('exportLotw') == 1) {
@@ -161,16 +195,45 @@ class adif extends CI_Controller {
$onlyop=null;
}
$data['qsos'] = $this->adif_data->export_custom($this->input->post('from'), $this->input->post('to'), $station_id, $exportLotw, $onlyop);
// Set headers for direct download
$filename = $this->session->userdata('user_callsign').'-'.date('Ymd-Hi').'.adi';
header('Content-Type: text/plain; charset=utf-8');
header('Content-Disposition: attachment; filename="'.$filename.'"');
$this->load->view('adif/data/exportall', $data);
echo $this->adifhelper->getAdifHeader($this->config->item('app_name'),$this->optionslib->get_option('version'));
// Collect QSO IDs for LoTW marking (since we can't access all at once)
$qso_ids_for_lotw = [];
if ((clubaccess_check(9)) && ($this->input->post('markLotw') == 1)) { // Only allow marking if clubofficer or regular user!
foreach ($data['qsos']->result() as $qso) {
$this->adif_data->mark_lotw_sent($qso->COL_PRIMARY_KEY);
// Stream QSOs in 5K chunks
$offset = 0;
$chunk_size = 5000;
do {
$qsos = $this->adif_data->export_custom_chunked($from, $to, $station_id, $exportLotw, $onlyop, $offset, $chunk_size);
if ($qsos && $qsos->num_rows() > 0) {
foreach ($qsos->result() as $qso) {
echo $this->adifhelper->getAdifLine($qso);
// Collect IDs for LoTW marking
$qso_ids_for_lotw[] = $qso->COL_PRIMARY_KEY;
}
// Free memory
$qsos->free_result();
}
$offset += $chunk_size;
} while ($qsos && $qsos->num_rows() > 0);
// Handle LoTW marking after export
if ((clubaccess_check(9)) && ($this->input->post('markLotw') == 1) && !empty($qso_ids_for_lotw)) {
foreach ($qso_ids_for_lotw as $qso_id) {
$this->adif_data->mark_lotw_sent($qso_id);
}
}
// Stop execution
exit;
}
public function mark_lotw() {

73
application/controllers/Api.php
View File

@@ -440,33 +440,64 @@ class API extends CI_Controller {
//load adif data module
$this->load->model('adif_data');
$this->load->library('AdifHelper');
//get qso data
$data['qsos'] = $this->adif_data->export_past_id($station_id, $fetchfromid, $limit);
// Initialize tracking variables
$total_fetched = 0;
$all_qso_ids = [];
$lastfetchedid = $fetchfromid;
//set internalonly attribute for adif creation
$data['internalrender'] = true;
// Process in chunks to avoid memory issues
$chunk_size = 5000;
$remaining_limit = $limit;
$offset = 0;
//if no new QSOs are ready, return that
$qso_count = count($data['qsos']->result());
if($qso_count <= 0) {
// Start building ADIF content
$adif_content = $this->adifhelper->getAdifHeader($this->config->item('app_name'),$this->optionslib->get_option('version'));
do {
// Calculate chunk size for this iteration
$current_chunk_size = min($chunk_size, $remaining_limit);
// Fetch chunk
$qsos = $this->adif_data->export_past_id_chunked($station_id, $fetchfromid, $current_chunk_size, null, $offset, $current_chunk_size);
if ($qsos && $qsos->num_rows() > 0) {
// Process chunk
foreach ($qsos->result() as $row) {
// Build ADIF content directly
$adif_content .= $this->adifhelper->getAdifLine($row);
// Track data for response
$all_qso_ids[] = $row->COL_PRIMARY_KEY;
$lastfetchedid = max($lastfetchedid, $row->COL_PRIMARY_KEY);
$total_fetched++;
}
// Free memory
$qsos->free_result();
// Update tracking
$remaining_limit -= $qsos->num_rows();
$offset += $qsos->num_rows();
// Stop if we've hit the requested limit
if ($total_fetched >= $limit) {
break;
}
}
// Continue if we got a full chunk and haven't hit the limit
} while ($qsos && $qsos->num_rows() > 0 && $total_fetched < $limit);
// Return response (same format as original)
if($total_fetched <= 0) {
http_response_code(200);
echo json_encode(['status' => 'successfull', 'message' => 'No new QSOs available.', 'lastfetchedid' => $fetchfromid, 'exported_qsos' => 0, 'adif' => null]);
return;
} else {
http_response_code(200);
echo json_encode(['status' => 'successfull', 'message' => 'Export successfull', 'lastfetchedid' => $lastfetchedid, 'exported_qsos' => $total_fetched, 'adif' => $adif_content]);
}
//convert data to ADIF
$adif_content = $this->load->view('adif/data/exportall', $data, TRUE);
//get new goalpost
$lastfetchedid = 0;
foreach ($data['qsos']->result() as $row) {
$lastfetchedid = max($lastfetchedid, $row->COL_PRIMARY_KEY);
}
//return API result
http_response_code(200);
echo json_encode(['status' => 'successfull', 'message' => 'Export successfull', 'lastfetchedid' => $lastfetchedid, 'exported_qsos' => $qso_count, 'adif' => $adif_content]);
}

40
application/controllers/Backup.php
View File

@@ -29,31 +29,39 @@ class Backup extends CI_Controller {
$clean_key = $this->security->xss_clean($key);
$this->load->helper('file');
$this->load->library('AdifHelper');
// Set memory limit to unlimited to allow heavy usage
ini_set('memory_limit', '-1');
$this->load->model('adif_data');
$filename = 'backup/logbook'. date('_Y_m_d_H_i_s') .'.adi';
$data['qsos'] = $this->adif_data->export_all($clean_key);
header('Content-Type: text/plain; charset=utf-8');
header('Content-Disposition: attachment; filename="'.$filename.'"');
$data['filename'] = 'backup/logbook'. date('_Y_m_d_H_i_s') .'.adi';
if ( ! write_file($data['filename'], $this->load->view('backup/exportall', $data, true)))
{
$data['status'] = false;
}
else
{
$data['status'] = true;
}
// Output ADIF header // No chance to use exportall-view any longer, because of chunking logic
echo $this->adifhelper->getAdifHeader($this->config->item('app_name'),$this->optionslib->get_option('version'));
$data['page_title'] = __("ADIF - Backup");
// Stream QSOs in 5K chunks
$offset = 0;
$chunk_size = 5000;
$this->load->view('interface_assets/header', $data);
$this->load->view('backup/adif_view');
$this->load->view('interface_assets/footer');
do {
$qsos = $this->adif_data->export_all_chunked($clean_key, null, null, false, null, $offset, $chunk_size);
if ($qsos->num_rows() > 0) {
foreach ($qsos->result() as $qso) {
echo $this->adifhelper->getAdifLine($qso);
}
// Free memory
$qsos->free_result();
}
$offset += $chunk_size;
} while ($qsos->num_rows() > 0);
exit;
}
/* Export the notes to XML */

9
application/libraries/AdifHelper.php
View File

@@ -262,6 +262,15 @@ class AdifHelper {
return $line;
}
function getAdifHeader($app_name,$version) {
$adif_header = "Wavelog ADIF export\n";
$adif_header .= "<ADIF_VER:5>3.1.6\n";
$adif_header .= "<PROGRAMID:".strlen($app_name).">".$app_name."\r\n";
$adif_header .= "<PROGRAMVERSION:".strlen($version).">".$version."\r\n";
$adif_header .= "<EOH>\n\n";
return $adif_header;
}
function getAdifFieldLine($adifcolumn, $dbvalue) {
if ($dbvalue !== "" && $dbvalue !== null && $dbvalue !== 0) {
return "<" . $adifcolumn . ":" . mb_strlen($dbvalue, "UTF-8") . ">" . $dbvalue . "\r\n";

109
application/models/Adif_data.php
View File

@@ -2,6 +2,48 @@
class adif_data extends CI_Model {
function export_all_chunked($api_key = null, $from = null, $to = null, $exportLotw = false, $onlyop = null, $offset = 0, $limit = 5000) {
$this->load->model('logbooks_model');
if ($api_key != null) {
$this->load->model('api_model');
if (strpos($this->api_model->access($api_key), 'r') !== false) {
$this->api_model->update_last_used($api_key);
$user_id = $this->api_model->key_userid($api_key);
$logbooks_locations_array = $this->list_station_locations($user_id);
}
} else {
$this->load->model('stations');
$logbooks_locations_array = $this->list_station_locations($this->session->userdata('user_id'));
}
$this->db->select($this->config->item('table_name').'.*, station_profile.*, dxcc_entities.name as station_country');
$this->db->order_by("COL_TIME_ON", "ASC");
$this->db->join('station_profile', 'station_profile.station_id = '.$this->config->item('table_name').'.station_id');
if ($from) {
$this->db->where("date(".$this->config->item('table_name').".COL_TIME_ON) >= ", $from);
}
if ($to) {
$this->db->where("date(".$this->config->item('table_name').".COL_TIME_ON) <= ",$to);
}
if ($onlyop) {
$this->db->where("upper(".$this->config->item('table_name').".col_operator)",$onlyop);
}
if ($exportLotw) {
$this->db->group_start();
$this->db->where($this->config->item('table_name').".COL_LOTW_QSL_SENT != 'Y'");
$this->db->or_where($this->config->item('table_name').".COL_LOTW_QSL_SENT", NULL);
$this->db->group_end();
}
$this->db->where_in('station_profile.station_id', $logbooks_locations_array);
$this->db->join('dxcc_entities', 'station_profile.station_dxcc = dxcc_entities.adif');
// Add chunking
$this->db->limit($limit, $offset);
$query = $this->db->get($this->config->item('table_name'));
return $query;
}
function export_all($api_key = null,$from = null, $to = null, $exportLotw = false, $onlyop = null) {
$this->load->model('logbooks_model');
if ($api_key != null) {
@@ -125,6 +167,52 @@ class adif_data extends CI_Model {
return $this->db->get();
}
function export_custom_chunked($from, $to, $station_id, $exportLotw = false, $onlyop = null, $offset = 0, $limit = 5000) {
// Copy export_custom logic but add chunking for station_id > 0
$this->load->model('Stations');
if ($station_id == 0) {
// Use existing chunked export_all for all stations
return $this->export_all_chunked(null, $from, $to, $exportLotw, $onlyop, $offset, $limit);
}
// Check station access
if (!$this->Stations->check_station_is_accessible($station_id)) {
return;
}
// Build query identical to export_custom but add LIMIT/OFFSET
$this->db->select(''.$this->config->item('table_name').'.*, station_profile.*, dxcc_entities.name as station_country');
$this->db->from($this->config->item('table_name'));
$this->db->where($this->config->item('table_name').'.station_id', $station_id);
// Apply same filters as export_custom
if ($from) {
$this->db->where("date(".$this->config->item('table_name').".COL_TIME_ON) >= ", $from);
}
if ($to) {
$this->db->where("date(".$this->config->item('table_name').".COL_TIME_ON) <= ",$to);
}
if ($onlyop) {
$this->db->where("upper(".$this->config->item('table_name').".col_operator)",$onlyop);
}
if ($exportLotw) {
$this->db->group_start();
$this->db->where($this->config->item('table_name').".COL_LOTW_QSL_SENT != 'Y'");
$this->db->or_where($this->config->item('table_name').".COL_LOTW_QSL_SENT", NULL);
$this->db->group_end();
}
$this->db->order_by($this->config->item('table_name').".COL_TIME_ON", "ASC");
$this->db->join('station_profile', 'station_profile.station_id = '.$this->config->item('table_name').'.station_id');
$this->db->join('dxcc_entities', 'station_profile.station_dxcc = dxcc_entities.adif', 'left outer');
// Add chunking
$this->db->limit($limit, $offset);
return $this->db->get();
}
function export_custom($from, $to, $station_id, $exportLotw = false, $onlyop = null) {
// be sure that station belongs to user
$this->load->model('Stations');
@@ -165,25 +253,24 @@ class adif_data extends CI_Model {
}
}
function export_past_id($station_id, $fetchfromid, $limit, $onlyop = null) {
//create query
function export_past_id_chunked($station_id, $fetchfromid, $limit, $onlyop = null, $offset = 0, $chunk_size = 5000) {
// Copy export_past_id logic but add chunking support
$this->db->select(''.$this->config->item('table_name').'.*, station_profile.*, dxcc_entities.name as station_country');
$this->db->from($this->config->item('table_name'));
$this->db->where($this->config->item('table_name').'.station_id', $station_id);
$this->db->where($this->config->item('table_name').".COL_PRIMARY_KEY > " , $fetchfromid); //only get values past the fetchfromid value
$this->db->order_by($this->config->item('table_name').".COL_TIME_ON", "ASC");
$this->db->join('station_profile', 'station_profile.station_id = '.$this->config->item('table_name').'.station_id');
$this->db->join('dxcc_entities', 'station_profile.station_dxcc = dxcc_entities.adif', 'left outer');
$this->db->where($this->config->item('table_name').".COL_PRIMARY_KEY > ", $fetchfromid);
if ($onlyop) {
$this->db->where("upper(".$this->config->item('table_name').".col_operator)",$onlyop);
}
// Add chunking
$this->db->limit($chunk_size, $offset);
$this->db->order_by("COL_PRIMARY_KEY", "ASC");
$this->db->join('station_profile', 'station_profile.station_id = '.$this->config->item('table_name').'.station_id');
$this->db->join('dxcc_entities', 'station_profile.station_dxcc = dxcc_entities.adif', 'left outer');
if ($limit > -1) {
$this->db->limit($limit);
}
//return result
return $this->db->get();
}

11
application/views/adif/data/exportall.php
View File

@@ -5,19 +5,14 @@
header('Content-Type: text/plain; charset=utf-8');
header('Content-Disposition: attachment; filename="'.$this->session->userdata('user_callsign').'-'.date('Ymd-Hi').'.adi"');
}
?>
Wavelog ADIF export
<ADIF_VER:5>3.1.6
<PROGRAMID:<?php echo strlen($this->config->item('app_name')); ?>><?php echo $this->config->item('app_name')."\r\n"; ?>
<PROGRAMVERSION:<?php echo strlen($this->optionslib->get_option('version')); ?>><?php echo $this->optionslib->get_option('version')."\r\n"; ?>
<EOH>
<?php
$CI =& get_instance();
if (!$CI->load->is_loaded('AdifHelper')) {
$CI->load->library('AdifHelper');
}
echo $CI->adifhelper->getAdifHeader($CI->config->item('app_name'),$CI->optionslib->get_option('version'));
foreach ($qsos->result() as $qso) {
echo $CI->adifhelper->getAdifLine($qso);
}
?>