CSV trace import

Software/PC_Application/SpectrumAnalyzer/tracewidgetsa.cpp

@@ -1,6 +1,7 @@
 #include "tracewidgetsa.h"
-
 #include "Traces/tracecsvexport.h"
+#include <QFileDialog>
+#include "Traces/traceimportdialog.h"
 
 TraceWidgetSA::TraceWidgetSA(TraceModel &model, QWidget *parent)
     : TraceWidget(model, parent)
@@ -13,3 +14,26 @@ void TraceWidgetSA::exportDialog()
     auto csv = new TraceCSVExport(model);
     csv->show();
 }
+
+void TraceWidgetSA::importDialog()
+{
+    auto filename = QFileDialog::getOpenFileName(nullptr, "Open measurement file", "", "CSV files (*.csv)", nullptr, QFileDialog::DontUseNativeDialog);
+    if (!filename.isEmpty()) {
+        std::vector<Trace*> traces;
+        QString prefix = QString();
+        auto csv = CSV::fromFile(filename);
+        traces = Trace::createFromCSV(csv);
+        // contruct prefix from filename
+        prefix = filename;
+        // remove any directory names (keep only the filename itself)
+        int lastSlash = qMax(prefix.lastIndexOf('/'), prefix.lastIndexOf('\\'));
+        if(lastSlash != -1) {
+            prefix.remove(0, lastSlash + 1);
+        }
+        // remove file type
+        prefix.truncate(prefix.indexOf('.'));
+        prefix.append("_");
+        auto i = new TraceImportDialog(model, traces, prefix);
+        i->show();
+    }
+}

Software/PC_Application/SpectrumAnalyzer/tracewidgetsa.h

@@ -9,7 +9,7 @@ public:
     TraceWidgetSA(TraceModel &model, QWidget *parent = nullptr);
 protected slots:
     virtual void exportDialog() override;
-    virtual void importDialog() override {};
+    virtual void importDialog() override;
 
 protected:
     virtual Trace::LiveParameter defaultParameter() override {return Trace::LiveParameter::Port1;};

Software/PC_Application/Traces/trace.cpp

@@ -14,8 +14,9 @@ Trace::Trace(QString name, QColor color, LiveParameter live)
       reflection(true),
       visible(true),
       paused(false),
-      touchstone(false),
+      createdFromFile(false),
       calibration(false),
+      timeDomain(false),
       lastMath(nullptr)
 {
     MathInfo self = {.math = this, .enabled = true};
@@ -24,6 +25,10 @@ Trace::Trace(QString name, QColor color, LiveParameter live)
 
     self.enabled = false;
     dataType = DataType::Frequency;
+    connect(this, &Trace::typeChanged, [=](){
+        dataType = timeDomain ? DataType::Time : DataType::Frequency;
+        emit outputTypeChanged(dataType);
+    });
 }
 
 Trace::~Trace()
@@ -97,14 +102,15 @@ void Trace::setName(QString name) {
     emit nameChanged();
 }
 
-void Trace::fillFromTouchstone(Touchstone &t, unsigned int parameter, QString filename)
+void Trace::fillFromTouchstone(Touchstone &t, unsigned int parameter)
 {
     if(parameter >= t.ports()*t.ports()) {
         throw runtime_error("Parameter for touchstone out of range");
     }
     clear();
-    setTouchstoneParameter(parameter);
+    timeDomain = false;
-    setTouchstoneFilename(filename);
+    fileParemeter = parameter;
+    filename = t.getFilename();
     for(unsigned int i=0;i<t.points();i++) {
         auto tData = t.point(i);
         Data d;
@@ -128,14 +134,84 @@ void Trace::fillFromTouchstone(Touchstone &t, unsigned int parameter, QString fi
     } else {
         reflection = false;
     }
-    touchstone = true;
+    createdFromFile = true;
     emit typeChanged(this);
     emit outputSamplesChanged(0, data.size());
 }
 
+QString Trace::fillFromCSV(CSV &csv, unsigned int parameter)
+{
+    // find correct column
+    unsigned int traceNum = 0;
+    vector<double> real;
+    vector<double> imag;
+    unsigned int i=1;
+    QString traceName;
+    bool hasImagValues;
+    for(;i<csv.columns();i++) {
+        traceName = QString();
+        hasImagValues = false;
+        // check column names
+        if(i < csv.columns() - 1) {
+            // not the last column, check if this and next header implies real/imag values
+            auto name_real = csv.getHeader(i);
+            auto name_imag = csv.getHeader(i + 1);
+            if(name_real.endsWith("_real") && name_imag.endsWith("_imag")) {
+                // check if headers have the same beginning
+                name_real.chop(5);
+                name_imag.chop(5);
+                if(name_real == name_imag) {
+                    hasImagValues = true;
+                    traceName = name_real;
+                }
+            }
+        }
+        if(!hasImagValues) {
+            traceName = csv.getHeader(i);
+        }
+        if(traceNum == parameter) {
+            // this is the desired trace
+            break;
+        } else {
+            traceNum++;
+        }
+        if(hasImagValues) {
+            // next column already used by this trace, skip
+            i++;
+        }
+    }
+    if(i >= csv.columns()) {
+        throw runtime_error("Not enough traces in CSV file");
+    }
+    real = csv.getColumn(i);
+    if(hasImagValues) {
+        imag = csv.getColumn(i + 1);
+    } else {
+        imag.resize(real.size());
+        fill(imag.begin(), imag.end(), 0.0);
+    }
+    clear();
+    fileParemeter = parameter;
+    filename = csv.getFilename();
+    auto xColumn = csv.getColumn(0);
+    timeDomain = csv.getHeader(0).compare("time", Qt::CaseInsensitive) == 0;
+    for(unsigned int i=0;i<xColumn.size();i++) {
+        Data d;
+        d.x = xColumn[i];
+        d.y = complex<double>(real[i], imag[i]);
+        addData(d);
+    }
+    reflection = false;
+    createdFromFile = true;
+    emit typeChanged(this);
+    emit outputSamplesChanged(0, data.size());
+    return traceName;
+}
+
 void Trace::fromLivedata(Trace::LivedataType type, LiveParameter param)
 {
-    touchstone = false;
+    timeDomain = false;
+    createdFromFile = false;
     _liveType = type;
     _liveParam = param;
     if(param == LiveParameter::S11 || param == LiveParameter::S22) {
@@ -211,10 +287,10 @@ nlohmann::json Trace::toJSON()
         j["parameter"] = _liveParam;
         j["livetype"] = _liveType;
         j["paused"] = paused;
-    } else if(isTouchstone()) {
+    } else if(isFromFile()) {
-        j["type"] = "Touchstone";
+        j["type"] = "File";
-        j["filename"] = touchstoneFilename.toStdString();
+        j["filename"] = filename.toStdString();
-        j["parameter"] = touchstoneParameter;
+        j["parameter"] = fileParemeter;
     }
     j["reflection"] = reflection;
     // TODO how to save assigned markers?
@@ -239,7 +315,7 @@ nlohmann::json Trace::toJSON()
 
 void Trace::fromJSON(nlohmann::json j)
 {
-    touchstone = false;
+    createdFromFile = false;
     calibration = false;
     _name = QString::fromStdString(j.value("name", "Missing name"));
     _color = QColor(QString::fromStdString(j.value("color", "yellow")));
@@ -249,15 +325,21 @@ void Trace::fromJSON(nlohmann::json j)
         _liveParam = j.value("parameter", LiveParameter::S11);
         _liveType = j.value("livetype", LivedataType::Overwrite);
         paused = j.value("paused", false);
-    } else if(type == "Touchstone") {
+    } else if(type == "Touchstone" || type == "File") {
-        std::string filename = j.value("filename", "");
+        auto filename = QString::fromStdString(j.value("filename", ""));
-        touchstoneParameter = j.value("parameter", 0);
+        fileParemeter = j.value("parameter", 0);
         try {
-            Touchstone t = Touchstone::fromFile(filename);
+            if(filename.endsWith(".csv")) {
-            fillFromTouchstone(t, touchstoneParameter, QString::fromStdString(filename));
+                auto csv = CSV::fromFile(filename);
+                fillFromCSV(csv, fileParemeter);
+            } else {
+                // has to be a touchstone file
+                Touchstone t = Touchstone::fromFile(filename.toStdString());
+                fillFromTouchstone(t, fileParemeter);
+            }
         } catch (const exception &e) {
             std::string what = e.what();
-            throw runtime_error("Failed to create touchstone:" + what);
+            throw runtime_error("Failed to create from file:" + what);
         }
     }
     reflection = j.value("reflection", false);
@@ -305,6 +387,46 @@ unsigned int Trace::toHash()
     return hash<std::string>{}(json_string);
 }
 
+std::vector<Trace *> Trace::createFromTouchstone(Touchstone &t)
+{
+    qDebug() << "Creating traces from touchstone...";
+    std::vector<Trace*> traces;
+    for(unsigned int i=0;i<t.ports()*t.ports();i++) {
+        auto trace = new Trace();
+        trace->fillFromTouchstone(t, i);
+        unsigned int sink = i / t.ports() + 1;
+        unsigned int source = i % t.ports() + 1;
+        trace->setName("S"+QString::number(sink)+QString::number(source));
+        traces.push_back(trace);
+    }
+    return traces;
+}
+
+std::vector<Trace *> Trace::createFromCSV(CSV &csv)
+{
+    qDebug() << "Creating traces from csv...";
+    std::vector<Trace*> traces;
+    auto n = csv.columns();
+    if(n >= 2) {
+        try {
+            // This will throw once no more column is available, can use infinite loop
+            unsigned int param = 0;
+            while(1) {
+                auto t = new Trace();
+                auto name = t->fillFromCSV(csv, param);
+                t->setName(name);
+                param++;
+                traces.push_back(t);
+            }
+        } catch (const exception &e) {
+            // nothing to do, this simply means we reached the end of the csv columns
+        }
+    } else {
+        qWarning() << "Unable to parse, not enough columns";
+    }
+    return traces;
+}
+
 void Trace::updateLastMath(vector<MathInfo>::reverse_iterator start)
 {
     TraceMath *newLast = nullptr;
@@ -332,6 +454,16 @@ void Trace::setReflection(bool value)
     reflection = value;
 }
 
+TraceMath::DataType Trace::outputType(TraceMath::DataType inputType)
+{
+    Q_UNUSED(inputType);
+    if(timeDomain) {
+        return DataType::Time;
+    } else {
+        return DataType::Frequency;
+    }
+}
+
 QString Trace::description()
 {
     return name() + ": measured data";
@@ -375,9 +507,9 @@ bool Trace::isPaused()
     return paused;
 }
 
-bool Trace::isTouchstone()
+bool Trace::isFromFile()
 {
-    return touchstone;
+    return createdFromFile;
 }
 
 bool Trace::isCalibration()
@@ -387,7 +519,7 @@ bool Trace::isCalibration()
 
 bool Trace::isLive()
 {
-    return !isCalibration() && !isTouchstone();
+    return !isCalibration() && !isFromFile();
 }
 
 bool Trace::isReflection()
@@ -594,19 +726,14 @@ Trace::Data Trace::sample(unsigned int index, SampleType type)
     return data;
 }
 
-QString Trace::getTouchstoneFilename() const
+QString Trace::getFilename() const
 {
-    return touchstoneFilename;
+    return filename;
 }
 
-void Trace::setTouchstoneFilename(const QString &value)
+unsigned int Trace::getFileParameter() const
 {
-    touchstoneFilename = value;
+    return fileParemeter;
-}
-
-unsigned int Trace::getTouchstoneParameter() const
-{
-    return touchstoneParameter;
 }
 
 double Trace::getNoise(double frequency)
@@ -633,8 +760,3 @@ int Trace::index(double x)
     }
     return lower - lastMath->rData().begin();
 }
-
-void Trace::setTouchstoneParameter(int value)
-{
-    touchstoneParameter = value;
-}

Software/PC_Application/Traces/trace.h

@@ -7,6 +7,7 @@
 #include <QColor>
 #include <set>
 #include "touchstone.h"
+#include "csv.h"
 #include "Device/device.h"
 #include "Math/tracemath.h"
 
@@ -43,7 +44,8 @@ public:
     void addData(const Data& d, const Protocol::SweepSettings& s);
     void addData(const Data& d, const Protocol::SpectrumAnalyzerSettings& s);
     void setName(QString name);
-    void fillFromTouchstone(Touchstone &t, unsigned int parameter, QString filename = QString());
+    void fillFromTouchstone(Touchstone &t, unsigned int parameter);
+    QString fillFromCSV(CSV &csv, unsigned int parameter); // returns the suggested trace name (not yet set in member data)
     void fromLivedata(LivedataType type, LiveParameter param);
     QString name() { return _name; };
     QColor color() { return _color; };
@@ -51,7 +53,7 @@ public:
     void pause();
     void resume();
     bool isPaused();
-    bool isTouchstone();
+    bool isFromFile();
     bool isCalibration();
     bool isLive();
     bool isReflection();
@@ -75,8 +77,8 @@ public:
     };
 
     Data sample(unsigned int index, SampleType type = SampleType::Frequency);
-    QString getTouchstoneFilename() const;
+    QString getFilename() const;
-    unsigned int getTouchstoneParameter() const;
+    unsigned int getFileParameter() const;
     /* Returns the noise in dbm/Hz for spectrum analyzer measurements. May return NaN if calculation not possible */
     double getNoise(double frequency);
     int index(double x);
@@ -84,7 +86,7 @@ public:
     void setCalibration(bool value);
     void setReflection(bool value);
 
-    DataType outputType(DataType inputType) override { Q_UNUSED(inputType) return DataType::Frequency;};
+    DataType outputType(DataType inputType) override;
     QString description() override;
 
     bool mathEnabled(); // check if math operations are enabled
@@ -119,9 +121,10 @@ public:
     // the trace can have (and its math function). It should not depend on the acquired trace samples
     unsigned int toHash();
 
+    static std::vector<Trace*> createFromTouchstone(Touchstone &t);
+    static std::vector<Trace*> createFromCSV(CSV &csv);
+
 public slots:
-    void setTouchstoneParameter(int value);
-    void setTouchstoneFilename(const QString &value);
     void setVisible(bool visible);
     void setColor(QColor color);
     void addMarker(TraceMarker *m);
@@ -146,10 +149,11 @@ private:
     bool reflection;
     bool visible;
     bool paused;
-    bool touchstone;
+    bool createdFromFile;
     bool calibration;
-    QString touchstoneFilename;
+    bool timeDomain;
-    unsigned int touchstoneParameter;
+    QString filename;
+    unsigned int fileParemeter;
     std::set<TraceMarker*> markers;
     struct {
         union {

Software/PC_Application/Traces/traceeditdialog.cpp

@@ -19,17 +19,17 @@ TraceEditDialog::TraceEditDialog(Trace &t, QWidget *parent) :
     ui->GSource->setId(ui->bLive, 0);
     ui->GSource->setId(ui->bFile, 1);
 
-    if(t.isCalibration()) {
+    if(t.isCalibration() || (t.isFromFile() && t.getFilename().endsWith(".csv"))) {
-        // prevent editing imported calibration traces
+        // prevent editing imported calibration traces (and csv files for now)
         ui->bLive->setEnabled(false);
         ui->bFile->setEnabled(false);
         ui->CLiveType->setEnabled(false);
         ui->CLiveParam->setEnabled(false);
     }
 
-    if(t.isTouchstone()) {
+    if(t.isFromFile() && !t.getFilename().endsWith(".csv")) {
         ui->bFile->click();
-        ui->touchstoneImport->setFile(t.getTouchstoneFilename());
+        ui->touchstoneImport->setFile(t.getFilename());
     }
 
     auto updateFileStatus = [this]() {
@@ -48,8 +48,8 @@ TraceEditDialog::TraceEditDialog(Trace &t, QWidget *parent) :
                     ui->CParameter->addItem(name);
                 }
             }
-            if(trace.getTouchstoneParameter() < touchstone.ports()*touchstone.ports()) {
+            if(trace.getFileParameter() < touchstone.ports()*touchstone.ports()) {
-                ui->CParameter->setCurrentIndex(trace.getTouchstoneParameter());
+                ui->CParameter->setCurrentIndex(trace.getFileParameter());
             } else {
                 ui->CParameter->setCurrentIndex(0);
             }
@@ -113,7 +113,7 @@ void TraceEditDialog::on_buttonBox_accepted()
         // only apply changes if it is not a calibration trace
         if (ui->bFile->isChecked()) {
             auto t = ui->touchstoneImport->getTouchstone();
-            trace.fillFromTouchstone(t, ui->CParameter->currentIndex(), ui->touchstoneImport->getFilename());
+            trace.fillFromTouchstone(t, ui->CParameter->currentIndex());
         } else {
             Trace::LivedataType type = Trace::LivedataType::Overwrite;
             Trace::LiveParameter param = Trace::LiveParameter::S11;

Software/PC_Application/Traces/tracemarker.cpp

@@ -942,6 +942,8 @@ void TraceMarker::update()
         setPosition(parentTrace->findExtremumFreq(true));
         helperMarkers[0]->setPosition(position + offset);
         break;
+    case Type::Last:
+        break;
     }
     emit dataChanged(this);
 }

Software/PC_Application/Traces/tracemodel.cpp

@@ -111,7 +111,7 @@ QVariant TraceModel::data(const QModelIndex &index, int role) const
         }
         break;
     case ColIndexPlayPause:
-        if (role == Qt::DecorationRole && !trace->isTouchstone()) {
+        if (role == Qt::DecorationRole && trace->isLive()) {
             if (trace->isPaused()) {
                 return QIcon(":/icons/pause.svg");
             } else {

Software/PC_Application/VNA/tracewidgetvna.cpp

@@ -42,27 +42,29 @@ TraceWidgetVNA::TraceWidgetVNA(TraceModel &model, QWidget *parent)
 
 void TraceWidgetVNA::importDialog()
 {
-    auto filename = QFileDialog::getOpenFileName(nullptr, "Open measurement file", "", "Touchstone files (*.s1p *.s2p *.s3p *.s4p)", nullptr, QFileDialog::DontUseNativeDialog);
+    auto filename = QFileDialog::getOpenFileName(nullptr, "Open measurement file", "", "Touchstone files (*.s1p *.s2p *.s3p *.s4p);;CSV files (*.csv)", nullptr, QFileDialog::DontUseNativeDialog);
-    if (filename.length() > 0) {
+    if (!filename.isEmpty()) {
-        auto t = Touchstone::fromFile(filename.toStdString());
         std::vector<Trace*> traces;
-        for(unsigned int i=0;i<t.ports()*t.ports();i++) {
+        QString prefix = QString();
-            auto trace = new Trace();
+        if(filename.endsWith(".csv")) {
-            trace->fillFromTouchstone(t, i, filename);
+            auto csv = CSV::fromFile(filename);
-            unsigned int sink = i / t.ports() + 1;
+            traces = Trace::createFromCSV(csv);
-            unsigned int source = i % t.ports() + 1;
+        } else {
-            trace->setName("S"+QString::number(sink)+QString::number(source));
+            // must be a touchstone file
-            traces.push_back(trace);
+            auto t = Touchstone::fromFile(filename.toStdString());
+            traces = Trace::createFromTouchstone(t);
         }
         // contruct prefix from filename
+        prefix = filename;
         // remove any directory names (keep only the filename itself)
-        int lastSlash = qMax(filename.lastIndexOf('/'), filename.lastIndexOf('\\'));
+        int lastSlash = qMax(prefix.lastIndexOf('/'), prefix.lastIndexOf('\\'));
         if(lastSlash != -1) {
-            filename.remove(0, lastSlash + 1);
+            prefix.remove(0, lastSlash + 1);
         }
         // remove file type
-        filename.truncate(filename.indexOf('.'));
+        prefix.truncate(prefix.indexOf('.'));
-        auto i = new TraceImportDialog(model, traces, filename+"_");
+        prefix.append("_");
+        auto i = new TraceImportDialog(model, traces, prefix);
         i->show();
     }
 }

Software/PC_Application/csv.cpp

@@ -43,6 +43,7 @@ CSV CSV::fromFile(QString filename, char sep)
             }
         }
     }
+    csv.filename = filename;
     return csv;
 }
 
@@ -69,6 +70,7 @@ void CSV::toFile(QString filename, char sep)
         file << endl;
     }
     file.close();
+    this->filename = filename;
 }
 
 std::vector<double> CSV::getColumn(QString header)
@@ -86,6 +88,11 @@ std::vector<double> CSV::getColumn(unsigned int index)
     return _columns.at(index).data;
 }
 
+QString CSV::getHeader(unsigned int index)
+{
+    return _columns.at(index).header;
+}
+
 void CSV::addColumn(QString name, const std::vector<double> &data)
 {
     Column c;
@@ -93,3 +100,13 @@ void CSV::addColumn(QString name, const std::vector<double> &data)
     c.data = data;
     _columns.push_back(c);
 }
+
+QString CSV::getFilename() const
+{
+    return filename;
+}
+
+void CSV::setFilename(const QString &value)
+{
+    filename = value;
+}

Software/PC_Application/csv.h

@@ -14,10 +14,14 @@ public:
     void toFile(QString filename, char sep = ',');
     std::vector<double> getColumn(QString header);
     std::vector<double> getColumn(unsigned int index);
+    QString getHeader(unsigned int index);
     unsigned int columns() { return _columns.size();}
 
     void addColumn(QString name, const std::vector<double> &data);
 
+    QString getFilename() const;
+    void setFilename(const QString &value);
+
 private:
     class Column {
     public:
@@ -25,6 +29,7 @@ private:
         std::vector<double> data;
     };
     std::vector<Column> _columns;
+    QString filename;
 };
 
 #endif // CSV_H

Software/PC_Application/touchstone.cpp

@@ -118,6 +118,7 @@ void Touchstone::toFile(string filename, Unit unit, Format format)
         }
     }
     file.close();
+    this->filename = QString::fromStdString(filename);
 }
 
 Touchstone Touchstone::fromFile(string filename)
@@ -274,6 +275,7 @@ Touchstone Touchstone::fromFile(string filename)
             }
         }
     }
+    ret.filename = QString::fromStdString(filename);
     return ret;
 }
 
@@ -365,3 +367,8 @@ void Touchstone::reduceTo1Port(unsigned int port)
     }
     m_ports = 1;
 }
+
+QString Touchstone::getFilename() const
+{
+    return filename;
+}

Software/PC_Application/touchstone.h

@@ -4,6 +4,7 @@
 #include <complex>
 #include <vector>
 #include <string>
+#include <QString>
 
 class Touchstone
 {
@@ -41,9 +42,12 @@ public:
     // remove all paramaters except the ones from port (port cnt starts at 0)
     void reduceTo1Port(unsigned int port);
     unsigned int ports() { return m_ports; }
+    QString getFilename() const;
+
 private:
     unsigned int m_ports;
     std::vector<Datapoint> m_datapoints;
+    QString filename;
 };
 
 #endif // TOUCHSTONE_H