Merge pull request #1222 from DB4SCW/cbr_import

Fix Contest data from third-party loggers via Cabrillo import
This commit is contained in:
Florian (DF2ET)
2025-01-22 12:11:37 +01:00
committed by GitHub
9 changed files with 661 additions and 6 deletions

View File

@@ -185,5 +185,7 @@ return array(
'ADI' => array('application/octet-stream','text/plain', 'audio/x-hx-aac-adif'),
'ADIF' => array('application/octet-stream','text/plain', 'audio/x-hx-aac-adif'),
'tq8' => 'application/octet-stream',
'TQ8' => 'application/octet-stream'
'TQ8' => 'application/octet-stream',
'cbr' => array('application/octet-stream','text/plain'),
'CBR' => array('application/octet-stream','text/plain')
);

View File

@@ -222,13 +222,31 @@ class adif extends CI_Controller {
$this->adif_parser->initialize();
$custom_errors = "";
$alladif=[];
$contest_qso_infos = [];
while($record = $this->adif_parser->get_record()) {
//overwrite the contest id if user chose a contest in UI
if ($contest != '') {
$record['contest_id']=$contest;
$record['contest_id'] = $contest;
}
//handle club operator
if ($club_operator != '') {
$record['operator']=strtoupper($club_operator);
$record['operator'] = strtoupper($club_operator);
}
//check if contest_id exists in record and extract all found contest_ids
if(array_key_exists('contest_id', $record)){
$contest_id = $record['contest_id'];
if($contest_id != ''){
if(array_key_exists($contest_id, $contest_qso_infos)){
$contest_qso_infos[$contest_id] += 1;
}else{
$contest_qso_infos[$contest_id] = 1;
}
}
}
if(count($record) == 0) {
break;
};
@@ -259,6 +277,7 @@ class adif extends CI_Controller {
log_message("Error","ADIF End");
$data['adif_errors'] = $custom_errors;
$data['skip_dupes'] = $this->input->post('skipDuplicate');
$data['imported_contests'] = $contest_qso_infos;
$data['page_title'] = __("ADIF Imported");
$this->load->view('interface_assets/header', $data);

View File

@@ -9,8 +9,8 @@ if ( ! defined('BASEPATH')) exit('No direct script access allowed');
class Cabrillo extends CI_Controller {
function __construct() {
parent::__construct();
public function __construct() {
parent::__construct();
$this->load->model('user_model');
if(!$this->user_model->authorize(2) || !clubaccess_check(9)) { $this->session->set_flashdata('error', __("You're not allowed to do that!")); redirect('dashboard'); }
@@ -139,4 +139,170 @@ class Cabrillo extends CI_Controller {
$this->session->set_flashdata('error', __("You're not allowed to do that!")); redirect('dashboard');
}
}
public function cbrimport(){
//load user stations
$this->load->model('stations');
$data['station_profile'] = $this->stations->all_of_user();
//set page title and target tab
$data['page_title'] = __("Cabrillo Import");
$data['tab'] = "cbr";
//configure upload
$config['upload_path'] = './uploads/';
$config['allowed_types'] = 'cbr|CBR|log|LOG';
//load upload library
$this->load->library('upload', $config);
//if upload fails, return with errors, reset upload filesize
if ( ! $this->upload->do_upload()) {
$data['error'] = $this->upload->display_errors();
$data['max_upload'] = ini_get('upload_max_filesize');
$this->load->view('interface_assets/header', $data);
$this->load->view('adif/import', $data);
$this->load->view('interface_assets/footer');
return;
}
//get data from upload
$contest_id = $this->input->post('contest_id', false) ?? '';
$data = array('upload_data' => $this->upload->data());
//set memory limit to allow big files
ini_set('memory_limit', '-1');
set_time_limit(0);
//load the logbook model
$this->load->model('logbook_model');
//load the Cabrillo parser
if (!$this->load->is_loaded('cbr_parser')) {
$this->load->library('cbr_parser');
}
//get flag about the presence of the serial number
$serial_number_present = ($this->input->post('serial_number_present', true) == 1);
//parse the uploaded file
$parsed_cbr = $this->cbr_parser->parse_from_file('./uploads/'.$data['upload_data']['file_name'], $serial_number_present);
//return with error, reset upload filesize
if(count($parsed_cbr["QSOS"]) < 1)
{
$data['error'] = __("Broken CBR file - no QSO data or incomplete header found.");
$data['max_upload'] = ini_get('upload_max_filesize');
//delete uploaded file
unlink('./uploads/' . $data['upload_data']['file_name']);
$this->load->view('interface_assets/header', $data);
$this->load->view('adif/import', $data);
$this->load->view('interface_assets/footer');
return;
}
//get all station ids for the active user
$this->load->model('stations');
$station_ids = [];
foreach ($this->stations->all_of_user()->result() as $station) {
array_push($station_ids, $station->station_id);
}
//overwrite contest id if chosen during upload
if($contest_id != ''){
$parsed_cbr["HEADER"]["CONTEST"] = $contest_id;
}
//create helper variables
$custom_errors = [];
$i = 1;
//process each contest qso
foreach ($parsed_cbr["QSOS"] as $qso) {
//get relevant data from header and qso line
$station_callsign = $parsed_cbr["HEADER"]["CALLSIGN"];
$contest_id = $parsed_cbr["HEADER"]["CONTEST"];
$callsign = $qso["RCVD_CALLSIGN"];
$band = $qso["BAND"];
$mode = $qso["MODE"];
$date = $qso["DATE"];
$time = $qso["TIME"];
//load QSO
$contest_qsos = $this->logbook_model->getContestQSO($station_ids, $station_callsign, $contest_id, $callsign, $band, $mode, $date, $time)->result();
//create error if more than 1 QSO is found and skip
if(count($contest_qsos) != 1){
array_push($custom_errors, sprintf(__("QSO %d not found or more than 1 QSO found that match the criteria of the CBR file. Skipping as a safety measure."), $i));
$i++;
continue;
}
//load the first and only row
$contest_qso = $contest_qsos[0];
//load unique primary key
$contest_qso_id = $contest_qso->COL_PRIMARY_KEY;
//get new serial numbers if required, otherwise default to null. If serial is not numeric, use 0
$stx = $serial_number_present ? (is_numeric($qso["SENT_SERIAL"]) ? (int)$qso["SENT_SERIAL"] : 0) : null;
$srx = $serial_number_present ? (is_numeric($qso["RCVD_SERIAL"]) ? (int)$qso["RCVD_SERIAL"] : 0) : null;
//get count of exchanges
$sent_exchange_count = $parsed_cbr["SENT_EXCHANGE_COUNT"];
$rcvd_exchange_count = $parsed_cbr["RCVD_EXCHANGE_COUNT"];
//default to empty exchange strings
$stxstring = null;
$srxstring = null;
//process all sent exchanges
for ($i=1; $i <= $sent_exchange_count; $i++) {
if($stxstring == null)
{
$stxstring = $qso["SENT_EXCH_" . $i];
}else{
$stxstring = $stxstring . ' ' . $qso["SENT_EXCH_" . $i];
}
}
//process all sent exchanges
for ($i=1; $i <= $rcvd_exchange_count; $i++) {
if($srxstring == null)
{
$srxstring = $qso["RCVD_EXCH_" . $i];
}else{
$srxstring = $srxstring . ' ' . $qso["RCVD_EXCH_" . $i];
}
}
//correct data on contest qso
$this->logbook_model->set_contest_fields($contest_qso_id, $stx, $stxstring, $srx, $srxstring);
//increment counter
$i++;
}
//delete uploaded file
unlink('./uploads/' . $data['upload_data']['file_name']);
//set data for view
$data['cbr_errors'] = $custom_errors;
$data['cbr_error_count'] = count($custom_errors);
$data['cbr_update_count'] = count($parsed_cbr["QSOS"]) - count($custom_errors);
$data['page_title'] = __("CBR Data Imported");
//get view to user
$this->load->view('interface_assets/header', $data);
$this->load->view('cabrillo/cbr_success');
$this->load->view('interface_assets/footer');
}
}

View File

@@ -0,0 +1,301 @@
<?php
class CBR_Parser
{
public function parse_from_file($filename, $serial_number_present = false) : array
{
//load file, call parser
return $this->parse(mb_convert_encoding(file_get_contents($filename), "UTF-8"), $serial_number_present);
}
public function parse(string $input, $serial_number_present = false) : array
{
//split the input into lines
$lines = explode("\n", trim($input));
//initialize the result array
$qso_lines_raw = [];
$header = [];
//helper variable to determine common 59 element indices in QSO lines
$common_59_indices = null;
//flag to indicate processing mode
$qso_mode = false;
//helper variable to determine the maximum number of qso data fields
$max_qso_fields = 0;
//loop through each line
foreach ($lines as $line) {
//if we encounter "QSO" or "X-QSO" switch processing mode to QSO mode
if (strpos($line, 'QSO:') === 0 or strpos($line, 'X-QSO:') === 0) {
$qso_mode = true;
}else {
$qso_mode = false;
}
//if we encounter "END-OF-LOG", stop processing lines
if (strpos($line, 'END-OF-LOG') === 0) {
break;
}
//process and collect header lines if qso mode is not set
if (!$qso_mode) {
//split the line into an array using ': ' as the delimiter
$parts = explode(': ', $line, 2);
//collect header information
$header[$parts[0]] = trim($parts[1]);
//skip to next line
continue;
}
//process and collect QSO lines if qso mode is set
if ($qso_mode) {
//split the line into the elements
$qso_elements = preg_split('/\s+/', trim($line));
//determine maximum qso field size
$max_qso_fields = max($max_qso_fields, count($qso_elements));
//add qso elements to qso line array
array_push($qso_lines_raw, $qso_elements);
//find all occurrences of "59"
$indices_of_59 = [];
foreach ($qso_elements as $index => $value) {
if ($value === "59" or $value === "599") {
$indices_of_59[] = $index;
}
}
//find common indices position
if ($common_59_indices === null) {
//initialize common indices on the first iteration
$common_59_indices = $indices_of_59;
} else {
//intersect with current indices, preserving only common indices
$common_59_indices = array_intersect($common_59_indices, $indices_of_59);
}
//skip to next line
continue;
}
}
//abort further processing if no qso lines were found, return header only
if(count($qso_lines_raw) < 1) {
$result = [];
$result["HEADER"] = $header;
$result["QSOS"] = [];
$result["SENT_59_POS"] = 0;
$result["RCVD_59_POS"] = 0;
$result["SENT_EXCHANGE_COUNT"] = 0;
$result["RCVD_EXCHANGE_COUNT"] = 0;
//return result
return $result;
}
//abort if basic things (Callsign and Contest ID) are not included in the header
$header_fields = array_keys($header);
if(!in_array('CALLSIGN', $header_fields) or !in_array('CONTEST', $header_fields)){
$result = [];
$result["HEADER"] = $header;
$result["QSOS"] = [];
$result["SENT_59_POS"] = 0;
$result["RCVD_59_POS"] = 0;
$result["SENT_EXCHANGE_COUNT"] = 0;
$result["RCVD_EXCHANGE_COUNT"] = 0;
//return blank result
return $result;
}
//get positions of 59s inside QSO lines
$sent_59_pos = min($common_59_indices);
$rcvd_59_pos = max($common_59_indices);
//get codeigniter instance
$CI = &get_instance();
//load Frequency library
if(!$CI->load->is_loaded('Frequency')) {
$CI->load->library('Frequency');
}
//using 59 positions, remake qso_lines
$qso_lines = [];
//change all QSOs into associative arrays with meaningful keys
foreach ($qso_lines_raw as $line) {
$qso_line = [];
//get well defined fields
$qso_line["QSO_MARKER"] = $line[0];
$qso_line["FREQ"] = $line[1];
$qso_line["CBR_MODE"] = $line[2];
$qso_line["DATE"] = $line[3];
$qso_line["TIME"] = $line[4];
$qso_line["OWN_CALLSIGN"] = $line[5];
$qso_line["SENT_59"] = $line[$sent_59_pos];
//set serial if requested
if($serial_number_present) {
$qso_line["SENT_SERIAL"] = $line[$sent_59_pos + 1];
}
//get all remaining sent exchanges
$exchange_nr = 1;
$startindex = ($sent_59_pos + ($serial_number_present ? 2 : 1));
$endindex = ($rcvd_59_pos - 1);
for ($i = $startindex; $i < $endindex; $i++) {
$qso_line["SENT_EXCH_" . $exchange_nr] = $line[$i];
$exchange_nr++;
}
//get rest of the well defined fields
$qso_line["RCVD_CALLSIGN"] = $line[$rcvd_59_pos - 1];
$qso_line["RCVD_59"] = $line[$rcvd_59_pos];
//set serial if requested
if($serial_number_present) {
$qso_line["RCVD_SERIAL"] = $line[$rcvd_59_pos + 1];
}
//get all remaining received exchanges
$exchange_nr = 1;
$startindex = ($rcvd_59_pos + ($serial_number_present ? 2 : 1));
$endindex = (count($line));
for ($i = $startindex; $i < $endindex; $i++) {
$qso_line["RCVD_EXCH_" . $exchange_nr] = $line[$i];
$exchange_nr++;
}
//end of data in CQR format
//enhance QSO data with additional fields
$band = "";
//convert frequency to integer if possible
if(is_numeric($qso_line["FREQ"])) {
$frequency = (int)$qso_line["FREQ"];
}else{
$frequency = null;
}
//convert CBR values to band where no real frequency is given.
//if frequency is given, consult the frequency library
switch ($qso_line["FREQ"]) {
case '50':
$band = '6m';
break;
case '70':
$band = '4m';
break;
case '144':
$band = '2m';
break;
case '222':
$band = '1.25m';
break;
case '432':
$band = '70cm';
break;
case '902':
$band = '33cm';
break;
case '1.2G':
$band = '23cm';
break;
case '2.3G':
$band = '13cm';
break;
case '3.4G':
$band = '9cm';
break;
case '5.7G':
$band = '6cm';
break;
case '10G':
$band = '3cm';
break;
case '24G':
$band = '1.25cm';
break;
case '47G':
$band = 'SAT';
break;
case '75G':
$band = 'SAT';
break;
case '122G':
$band = 'SAT';
break;
case '134G':
$band = 'SAT';
break;
case '241G':
$band = 'SAT';
break;
case 'LIGHT':
$band = 'SAT';
break;
default:
$band = $CI->frequency->GetBand($frequency * 1000);
break;
}
//set band data for QSO
$qso_line["BAND"] = $band;
//get Wavelog mode
$mode = "";
switch ($qso_line["CBR_MODE"]) {
case 'CW':
$mode = 'CW';
break;
case 'PH':
$mode = 'SSB';
break;
case 'FM':
$mode = 'FM';
break;
case 'RY':
$mode = 'RTTY';
break;
case 'DG':
//indeterminate Digimode
$mode = '';
break;
default:
//something is wrong with the CBR file
$mode = '';
break;
}
//set mode data for QSO
$qso_line["MODE"] = $mode;
//collect new associative array
array_push($qso_lines, $qso_line);
}
//construct result, including positions of 59s for further processing down the line
$result = [];
$result["HEADER"] = $header;
$result["QSOS"] = $qso_lines;
$result["SENT_59_POS"] = $sent_59_pos;
$result["RCVD_59_POS"] = $rcvd_59_pos;
$result["SENT_EXCHANGE_COUNT"] = $rcvd_59_pos - $sent_59_pos - ($serial_number_present ? 3 : 2);
$result["RCVD_EXCHANGE_COUNT"] = $max_qso_fields - 1 - $rcvd_59_pos - ($serial_number_present ? 1 : 0);
//return result
return $result;
}
}
?>

View File

@@ -5572,9 +5572,71 @@ class Logbook_model extends CI_Model {
}
return '';
}
function getContestQSO(array $station_ids, string $station_callsign, string $contest_id, string $callsign, string $band, string $mode, string $date, string $time)
{
//load QSO table
$this->db->select('*');
$this->db->from($this->config->item('table_name'));
//load only for given station_ids
$this->db->where_in('station_id', $station_ids);
//load only for the station_callsign given
$this->db->where('COL_STATION_CALLSIGN', xss_clean($station_callsign));
//load only for the given contest id
$this->db->where('COL_CONTEST_ID', xss_clean($contest_id));
//load only for this qso partners callsign
$this->db->where('COL_CALL', xss_clean($callsign));
//load only for given band (no cleaning necessary because provided by wavelog itself)
$this->db->where('COL_BAND', $band);
//load only for specific mode if the mode is determinate. If not, omit it. In most cases, that should be fine. Also provided by wavelog itself, so no cleaning.
if($mode != '') {
$this->db->where('COL_MODE', $mode);
}
//prepare datetime from format '2099-12-31 13:47' to be usable in a performant query
$datetime_raw = $date . ' ' . substr($time, 0, 2) . ':' . substr($time, 2, 2);
$datetime = new DateTime($datetime_raw,new DateTimeZone('UTC'));
$from_datetime = $datetime->format('Y-m-d H:i:s');
$datetime->add(new DateInterval('PT1M'));
$to_datetime = $datetime->format('Y-m-d H:i:s');
//load only QSOs during this minute
$this->db->where('COL_TIME_ON >=', $from_datetime);
$this->db->where('COL_TIME_ON <', $to_datetime);
//return whatever is left
return $this->db->get();
}
function set_contest_fields($qso_primary_key, ?int $stx, ?string $stxstring, ?int $srx, ?string $srxstring) {
//assemble data fields from input
$data = $data = array(
'COL_STX' => $stx,
'COL_STX_STRING' => $stxstring == null ? null : substr($stxstring, 0, 32),
'COL_SRX' => $srx,
'COL_SRX_STRING' => $srxstring == null ? null : substr($srxstring, 0, 32)
);
//narrow db operation down to 1 QSO
$this->db->where(array('COL_PRIMARY_KEY' => $qso_primary_key));
//update data and return
$this->db->update($this->config->item('table_name'), $data);
return;
}
}
function validateADIFDate($date, $format = 'Ymd') {
$d = DateTime::createFromFormat($format, $date);
return $d && $d->format($format) == $date;
}

View File

@@ -13,7 +13,7 @@
</div>
<div class="card-body">
<?php if($dcl_error_count[0] > 0) { ?>
<h3 class="card-title">Yay, its updated!</h3>
<h3 class="card-title"><?= __("Yay, its updated!"); ?></h3>
<p class="card-text"><?= __("DCL information for DOKs has been updated.")?></p>
<?php } else { ?>
<h3 class="card-title"><?= __("No QSOs found which could be updated.")?></h3>

View File

@@ -35,6 +35,15 @@
echo 'false';
} ?>"><?= __("DARC DCL") ?></a>
</li>
<li class="nav-item">
<a class="nav-link <?php if ($showtab == 'cbr') {
echo 'active';
} ?>" id="cbr-tab" data-bs-toggle="tab" href="#cbr" role="tab" aria-controls="cbr" aria-selected="<?php if ($showtab == 'cbr') {
echo 'true';
} else {
echo 'false';
} ?>"><?= __("CBR Import") ?></a>
</li>
</ul>
</div>
@@ -314,6 +323,44 @@
</form>
</div>
</div>
<div class="tab-pane <?php if ($showtab == 'cbr') {
echo 'active';
} else {
echo 'fade';
} ?>" id="cbr" role="tabpanel" aria-labelledby="home-tab">
<?php if (isset($error) && $showtab == 'cbr') { ?>
<div class="alert alert-danger" role="alert">
<?php echo $error; ?>
</div>
<?php } ?>
<p class="card-text"><span class="badge text-bg-info"><?= __("Information"); ?></span> <?= __("If you imported an ADIF file of a contest, provided by another logging software, sometimes, depending on that software, your exchanges will not be imported properly from that softwares ADIF. If you like to correct that, you can provide the Cabrillo file that this software also provides to rewrite that data in Wavelog.") ?></p>
<p class="card-text"><span class="badge text-bg-warning"><?= __("Important"); ?></span> <?= __("Please use this function before changing anything about the QSOs in Wavelog, as this function uses the Contest ID, as well as date and time information from both your already imported ADIF file, as well as the CBR file you are about to upload to match the QSOs and only correct relevant data.") ?></p>
<form class="form" action="<?php echo site_url('cabrillo/cbrimport'); ?>" method="post" enctype="multipart/form-data">
<div class="mb-3 row">
<div class="col-md-10">
<div class="small form-text text-muted"><span class="badge text-bg-success"><?= __("Optional"); ?></span><?= __(" Contest Name, only if Contest ID in CBR is different") ?></div>
<select name="contest_id" id="contest_id" class="form-select mb-2 me-sm-2 w-50 w-lg-100">
<option value="" selected><?= __("No Contest"); ?></option>
<?php
foreach ($contests as $contest) {
echo '<option value="' . $contest['adifname'] . '">' . $contest['name'] . '</option>';
} ?>
</select>
<div class="form-check-inline">
<input class="form-check-input" type="checkbox" name="serial_number_present" value="1" id="serial_number_present" unchecked>
<label class="form-check-label" for="serial_number_present"><?= __("A serial number is ALWAYS part of the exchange for both parties in this contest.") ?></label>
</div>
<div class="small form-text text-muted"><?= __("If you or your partner only sometimes exchange serial numbers, please leave this unchecked.") ?></div>
<div class="small form-text text-muted"><?= __("If unchecked, this will erase the default serial number that (for example) N1MM+ produces. If checked, it will correct the serial number if necessary.") ?></div>
</div>
</div>
<input class="form-control w-auto mb-2 me-sm-2" type="file" name="userfile" size="20" />
<button type="submit" class="btn btn-sm btn-primary mb-2" value="Upload"><?= __("Upload") ?></button>
</form>
</div>
</div>
</div>
</div>
</div>

View File

@@ -12,14 +12,34 @@
<?= __("ADIF Imported")?>
</div>
<div class="card-body">
<!-- Success message -->
<h3 class="card-title"><?= __("Yay, its imported!")?></h3>
<p class="card-text"><?= __("The ADIF File has been imported.")?>
<!-- Dupe information -->
<?php if(isset($skip_dupes)) {
echo " <b>" . __("Dupes were inserted!") . "</b>";
} else {
echo " ".__("Dupes were skipped.");
} ?>
</p>
<!-- Display imported information for contest data fixing if contest data was imported -->
<?php if(count($imported_contests) > 0) {?>
<p style="color:red;"><?= __("You imported at least 1 QSO containing a contest ID.")?></p>
<p><?= __("Sometimes, depending on your contest logging software, your exchanges will not be imported properly from that softwares ADIF. If you like to correct that, switch to the CBR Import Tab of the ADIF Import page.")?></p>
<p><?= __("We found the following numbers of QSOs for the following contest IDs:")?></p>
<!-- List imported contest data -->
<ul>
<?php foreach ($imported_contests as $contestid => $qsocount) { ?>
<li><?php echo $contestid . ' (' . $qsocount . ' '. ($qsocount == 1 ? 'QSO' : 'QSOs') .')'; ?></li>
<?php } ?>
</ul>
<?php } ?>
<!-- Display errors for ADIF import -->
<?php if($adif_errors) { ?>
<h3><?= __("Import details / possible problems")?></h3>
<p><?= __("You might have ADIF errors, the QSOs have still been added. Please check the following information:")?></p>

View File

@@ -0,0 +1,38 @@
<div class="container">
<br>
<?php if($this->session->flashdata('message')) { ?>
<!-- Display Message -->
<div class="alert-message error">
<p><?php echo $this->session->flashdata('message'); ?></p>
</div>
<?php } ?>
<div class="card">
<div class="card-header">
<?= __("Results of CBR Contest Data Update");?>
</div>
<div class="card-body">
<?php if($cbr_update_count > 0) { ?>
<h3 class="card-title"><?= __("Yay, its updated!"); ?></h3>
<p class="card-text"><?= __("Your contest QSOs have been updated using the values of your Cabrillo file.")?></p>
<?php } else { ?>
<h3 class="card-title"><?= __("No QSOs were updated by your Cabrillo file.")?></h3>
<?php } ?>
<div class="alert alert-info" role="alert">
<?= __("QSOs updated")?>: <?php echo $cbr_update_count ?> / <?= __("QSOs ignored")?>: <?php echo $cbr_error_count ?>
</div>
<?php if($cbr_error_count > 0) { ?>
<h3><?= __("CBR errors")?></h3>
<table width="100%">
<?php foreach ($cbr_errors as $error) { ?>
<tr>
<td><?php echo $error; ?></td>
</tr>
<?php } ?>
</table>
<?php } ?>
</div>
</div>
</div>