Merge branch 'q3k/treemodel-fast' into 'master'

gui: lazy loading tree model

See merge request SymbioticEDA/nextpnr!25

gui/designwidget.cc

@@ -34,7 +34,7 @@ DesignWidget::DesignWidget(QWidget *parent) : QWidget(parent), ctx(nullptr), sel
 {
     // Add tree view
     treeView = new QTreeView();
-    treeModel = new ContextTreeModel();
+    treeModel = new TreeModel::Model();
     treeView->setModel(treeModel);
     treeView->setContextMenuPolicy(Qt::CustomContextMenu);
     treeView->setSelectionMode(QAbstractItemView::ExtendedSelection);
@@ -215,7 +215,11 @@ void DesignWidget::newContext(Context *ctx)
 
     highlightSelected.clear();
     this->ctx = ctx;
-    treeModel->loadData(ctx);
+    {
+        std::lock_guard<std::mutex> lock_ui(ctx->ui_mutex);
+        std::lock_guard<std::mutex> lock(ctx->mutex);
+        treeModel->loadContext(ctx);
+    }
     updateTree();
 }
 
@@ -223,9 +227,9 @@ void DesignWidget::updateTree()
 {
     clearProperties();
 
-    QMap<ContextTreeItem *, int>::iterator i = highlightSelected.begin();
+    QMap<TreeModel::Item *, int>::iterator i = highlightSelected.begin();
     while (i != highlightSelected.end()) {
-        QMap<ContextTreeItem *, int>::iterator prev = i;
+        QMap<TreeModel::Item *, int>::iterator prev = i;
         ++i;
         if (prev.key()->type() == ElementType::NET && ctx->nets.find(prev.key()->id()) == ctx->nets.end()) {
             highlightSelected.erase(prev);
@@ -235,7 +239,11 @@ void DesignWidget::updateTree()
         }
     }
 
-    treeModel->updateData(ctx);
+    {
+        std::lock_guard<std::mutex> lock_ui(ctx->ui_mutex);
+        std::lock_guard<std::mutex> lock(ctx->mutex);
+        treeModel->updateCellsNets(ctx);
+    }
 }
 QtProperty *DesignWidget::addTopLevelProperty(const QString &id)
 {
@@ -309,24 +317,33 @@ QtProperty *DesignWidget::addSubGroup(QtProperty *topItem, const QString &name)
 
 void DesignWidget::onClickedBel(BelId bel, bool keep)
 {
-    ContextTreeItem *item = treeModel->nodeForIdType(ElementType::BEL, ctx->getBelName(bel).c_str(ctx));
+    auto item = treeModel->nodeForIdType(ElementType::BEL, ctx->getBelName(bel));
-    selectionModel->setCurrentIndex(treeModel->indexFromNode(item),
+    if (!item)
+        return;
+
+    selectionModel->setCurrentIndex(treeModel->indexFromNode(*item),
                                     keep ? QItemSelectionModel::Select : QItemSelectionModel::ClearAndSelect);
     Q_EMIT selected(getDecals(ElementType::BEL, ctx->getBelName(bel)), keep);
 }
 
 void DesignWidget::onClickedWire(WireId wire, bool keep)
 {
-    ContextTreeItem *item = treeModel->nodeForIdType(ElementType::WIRE, ctx->getWireName(wire).c_str(ctx));
+    auto item = treeModel->nodeForIdType(ElementType::WIRE, ctx->getWireName(wire));
-    selectionModel->setCurrentIndex(treeModel->indexFromNode(item),
+    if (!item)
+        return;
+
+    selectionModel->setCurrentIndex(treeModel->indexFromNode(*item),
                                     keep ? QItemSelectionModel::Select : QItemSelectionModel::ClearAndSelect);
     Q_EMIT selected(getDecals(ElementType::WIRE, ctx->getWireName(wire)), keep);
 }
 
 void DesignWidget::onClickedPip(PipId pip, bool keep)
 {
-    ContextTreeItem *item = treeModel->nodeForIdType(ElementType::PIP, ctx->getPipName(pip).c_str(ctx));
+    auto item = treeModel->nodeForIdType(ElementType::PIP, ctx->getPipName(pip));
-    selectionModel->setCurrentIndex(treeModel->indexFromNode(item),
+    if (!item)
+        return;
+
+    selectionModel->setCurrentIndex(treeModel->indexFromNode(*item),
                                     keep ? QItemSelectionModel::Select : QItemSelectionModel::ClearAndSelect);
     Q_EMIT selected(getDecals(ElementType::PIP, ctx->getPipName(pip)), keep);
 }
@@ -339,7 +356,7 @@ void DesignWidget::onSelectionChanged(const QItemSelection &, const QItemSelecti
     if (selectionModel->selectedIndexes().size() > 1) {
         std::vector<DecalXY> decals;
         for (auto index : selectionModel->selectedIndexes()) {
-            ContextTreeItem *item = treeModel->nodeFromIndex(index);
+            TreeModel::Item *item = treeModel->nodeFromIndex(index);
             std::vector<DecalXY> d = getDecals(item->type(), item->id());
             std::move(d.begin(), d.end(), std::back_inserter(decals));
         }
@@ -349,7 +366,7 @@ void DesignWidget::onSelectionChanged(const QItemSelection &, const QItemSelecti
     QModelIndex index = selectionModel->selectedIndexes().at(0);
     if (!index.isValid())
         return;
-    ContextTreeItem *clickItem = treeModel->nodeFromIndex(index);
+    TreeModel::Item *clickItem = treeModel->nodeFromIndex(index);
 
     ElementType type = clickItem->type();
     if (type == ElementType::NONE)
@@ -597,7 +614,7 @@ std::vector<DecalXY> DesignWidget::getDecals(ElementType type, IdString value)
     return decals;
 }
 
-void DesignWidget::updateHighlightGroup(QList<ContextTreeItem *> items, int group)
+void DesignWidget::updateHighlightGroup(QList<TreeModel::Item *> items, int group)
 {
     const bool shouldClear = items.size() == 1;
     for (auto item : items) {
@@ -622,7 +639,7 @@ void DesignWidget::updateHighlightGroup(QList<ContextTreeItem *> items, int grou
 void DesignWidget::prepareMenuProperty(const QPoint &pos)
 {
     QTreeWidget *tree = propertyEditor->treeWidget();
-    QList<ContextTreeItem *> items;
+    QList<TreeModel::Item *> items;
     for (auto itemContextMenu : tree->selectedItems()) {
         QtBrowserItem *browserItem = propertyEditor->itemToBrowserItem(itemContextMenu);
         if (!browserItem)
@@ -632,11 +649,14 @@ void DesignWidget::prepareMenuProperty(const QPoint &pos)
         if (type == ElementType::NONE)
             continue;
         IdString value = ctx->id(selectedProperty->valueText().toStdString());
-        items.append(treeModel->nodeForIdType(type, value.c_str(ctx)));
+        auto node = treeModel->nodeForIdType(type, value);
+        if (!node)
+            continue;
+        items.append(*node);
     }
     int selectedIndex = -1;
     if (items.size() == 1) {
-        ContextTreeItem *item = items.at(0);
+        TreeModel::Item *item = items.at(0);
         if (highlightSelected.contains(item))
             selectedIndex = highlightSelected[item];
     }
@@ -677,13 +697,13 @@ void DesignWidget::prepareMenuTree(const QPoint &pos)
     if (selectionModel->selectedIndexes().size() == 0)
         return;
 
-    QList<ContextTreeItem *> items;
+    QList<TreeModel::Item *> items;
     for (auto index : selectionModel->selectedIndexes()) {
-        ContextTreeItem *item = treeModel->nodeFromIndex(index);
+        TreeModel::Item *item = treeModel->nodeFromIndex(index);
         items.append(item);
     }
     if (items.size() == 1) {
-        ContextTreeItem *item = items.at(0);
+        TreeModel::Item *item = items.at(0);
         if (highlightSelected.contains(item))
             selectedIndex = highlightSelected[item];
     }
@@ -709,9 +729,9 @@ void DesignWidget::onItemDoubleClicked(QTreeWidgetItem *item, int column)
 {
     QtProperty *selectedProperty = propertyEditor->itemToBrowserItem(item)->property();
     ElementType type = getElementTypeByName(selectedProperty->propertyId());
-    ContextTreeItem *it = treeModel->nodeForIdType(type, selectedProperty->valueText());
+    auto it = treeModel->nodeForIdType(type, ctx->id(selectedProperty->valueText().toStdString()));
     if (it)
-        selectionModel->setCurrentIndex(treeModel->indexFromNode(it), QItemSelectionModel::ClearAndSelect);
+        selectionModel->setCurrentIndex(treeModel->indexFromNode(*it), QItemSelectionModel::ClearAndSelect);
 }
 
 void DesignWidget::onDoubleClicked(const QModelIndex &index) { Q_EMIT zoomSelected(); }
@@ -723,6 +743,9 @@ void DesignWidget::onSearchInserted()
         if (currentIndex >= currentSearchIndexes.size())
             currentIndex = 0;
     } else {
+        std::lock_guard<std::mutex> lock_ui(ctx->ui_mutex);
+        std::lock_guard<std::mutex> lock(ctx->mutex);
+
         currentSearch = searchEdit->text();
         currentSearchIndexes = treeModel->search(searchEdit->text());
         currentIndex = 0;

gui/designwidget.h

@@ -51,7 +51,7 @@ class DesignWidget : public QWidget
     void updateButtons();
     void addToHistory(QModelIndex item);
     std::vector<DecalXY> getDecals(ElementType type, IdString value);
-    void updateHighlightGroup(QList<ContextTreeItem *> item, int group);
+    void updateHighlightGroup(QList<TreeModel::Item *> item, int group);
   Q_SIGNALS:
     void info(std::string text);
     void selected(std::vector<DecalXY> decal, bool keep);
@@ -77,7 +77,7 @@ class DesignWidget : public QWidget
 
     QTreeView *treeView;
     QItemSelectionModel *selectionModel;
-    ContextTreeModel *treeModel;
+    TreeModel::Model *treeModel;
     QLineEdit *searchEdit;
     QtVariantPropertyManager *variantManager;
     QtVariantPropertyManager *readOnlyManager;
@@ -99,7 +99,7 @@ class DesignWidget : public QWidget
     QAction *actionClear;
 
     QColor highlightColors[8];
-    QMap<ContextTreeItem *, int> highlightSelected;
+    QMap<TreeModel::Item *, int> highlightSelected;
 
     QString currentSearch;
     QList<QModelIndex> currentSearchIndexes;

gui/fpgaviewwidget.cc

@@ -323,14 +323,18 @@ void FPGAViewWidget::paintGL()
         flags = rendererData_->flags;
     }
 
-    {
-        QMutexLocker locker(&rendererArgsLock_);
-        rendererArgs_->flags.clear();
-    }
-
     // Check flags passed through pipeline.
     if (flags.zoomOutbound) {
-        zoomOutbound();
+        // If we're doing init zoomOutbound, make sure we're actually drawing
+        // something already.
+        if (rendererData_->gfxByStyle[GraphicElement::STYLE_FRAME].vertices.size() != 0) {
+            zoomOutbound();
+            flags.zoomOutbound = false;
+            {
+                QMutexLocker lock(&rendererArgsLock_);
+                rendererArgs_->flags.zoomOutbound = false;
+            }
+        }
     }
 }
 

gui/fpgaviewwidget.h

@@ -251,8 +251,6 @@ class FPGAViewWidget : public QOpenGLWidget, protected QOpenGLFunctions
             zoomOutbound = other.zoomOutbound;
             return *this;
         }
-
-        void clear() { zoomOutbound = false; }
     };
 
     struct RendererArgs

gui/treemodel.cc

@@ -2,6 +2,7 @@
  *  nextpnr -- Next Generation Place and Route
  *
  *  Copyright (C) 2018  Miodrag Milanovic <miodrag@symbioticeda.com>
+ *  Copyright (C) 2018  Serge Bazanski <q3k@symbioticeda.com>
  *
  *  Permission to use, copy, modify, and/or distribute this software for any
  *  purpose with or without fee is hereby granted, provided that the above
@@ -18,265 +19,250 @@
  */
 
 #include "treemodel.h"
+#include "log.h"
 
 NEXTPNR_NAMESPACE_BEGIN
 
-static bool contextTreeItemLessThan(const ContextTreeItem *v1, const ContextTreeItem *v2)
+namespace TreeModel {
- {
-     return v1->name() < v2->name();
- }
 
-ContextTreeItem::ContextTreeItem() { parentNode = nullptr; }
 
-ContextTreeItem::ContextTreeItem(QString name)
+// converts 'aa123bb432' -> ['aa', '123', 'bb', '432']
-        : parentNode(nullptr), itemId(IdString()), itemType(ElementType::NONE), itemName(name)
+std::vector<QString> IdStringList::alphaNumSplit(const QString &str)
 {
+    std::vector<QString> res;
+    QString current_part;
+
+    bool number = true;
+    for (const auto c : str) {
+        if (current_part.size() == 0 && res.size() == 0) {
+            current_part.push_back(c);
+            number = c.isNumber();
+            continue;
+        }
+
+        if (number != c.isNumber()) {
+            number = c.isNumber();
+            res.push_back(current_part);
+            current_part.clear();
+        }
+
+        current_part.push_back(c);
+    }
+    
+    res.push_back(current_part);
+
+    return res;
 }
 
-ContextTreeItem::ContextTreeItem(IdString id, ElementType type, QString name)
+void IdStringList::updateElements(Context *ctx, std::vector<IdString> elements)
-        : parentNode(nullptr), itemId(id), itemType(type), itemName(name)
 {
+    bool changed = false;
+
+    // For any elements that are not yet in managed_, created them.
+    std::unordered_set<IdString> element_set;
+    for (auto elem : elements) {
+        element_set.insert(elem);
+        auto existing = managed_.find(elem);
+        if (existing == managed_.end()) {
+            auto item = new IdStringItem(ctx, elem, this, child_type_);
+            managed_.emplace(elem, std::unique_ptr<IdStringItem>(item));
+            changed = true;
+        }
+    }
+    
+    // For any elements that are in managed_ but not in new, delete them.
+    for (auto &pair : managed_) {
+        if (element_set.count(pair.first) != 0) {
+            continue;
+        }
+        managed_.erase(pair.first);
+        changed = true;
+    }
+
+    // Return early if there are no changes.
+    if (!changed)
+        return;
+
+    // Rebuild children list.
+    children_.clear();
+    for (auto &pair : managed_) {
+        if (element_set.count(pair.first) != 0) {
+            children_.push_back(pair.second.get());
+        }
+    }
+
+    // Sort new children
+    qSort(children_.begin(), children_.end(), [&](const Item *a, const Item *b){
+        auto parts_a = alphaNumSplit(a->name());
+        auto parts_b = alphaNumSplit(b->name());
+
+        // Short-circuit for different part count.
+        if (parts_a.size() != parts_b.size()) {
+            return parts_a.size() < parts_b.size();
+        }
+
+        for (size_t i = 0; i < parts_a.size(); i++) {
+            auto &part_a = parts_a.at(i);
+            auto &part_b = parts_b.at(i);
+            
+            bool a_is_number, b_is_number;
+            int a_number = part_a.toInt(&a_is_number);
+            int b_number = part_b.toInt(&b_is_number);
+
+            // If both parts are numbers, compare numerically.
+            // If they're equal, continue to next part.
+            if (a_is_number && b_is_number) {
+                if (a_number != b_number) {
+                    return a_number < b_number;
+                } else {
+                    continue;
+                }
+            }
+
+            // For different alpha/nonalpha types, make numeric parts appear
+            // first.
+            if (a_is_number != b_is_number) {
+                return a_is_number;
+            }
+
+            // If both parts are numbers, compare lexically.
+            // If they're equal, continue to next part.
+            if (part_a == part_b) {
+                continue;
+            }
+            return part_a < part_b;
+        }
+
+        // Same string.
+        return true;
+    });
 }
 
-ContextTreeItem::~ContextTreeItem()
+void IdStringList::search(QList<Item*> &results, QString text, int limit)
 {
-    if (parentNode)
+    for (const auto &child : children_) {
-        parentNode->children.removeOne(this);
+        if (limit != -1 && results.size() > limit)
-    qDeleteAll(children);
+            return;
-}
+
-void ContextTreeItem::addChild(ContextTreeItem *item)
+        if (child->name().contains(text))
-{
+            results.push_back(child);
-    item->parentNode = this;
+    }
-    children.append(item);
 }
 
-void ContextTreeItem::sort()
-{
-    for (auto item : children)
-        if (item->count()>1) item->sort();
-    qSort(children.begin(), children.end(), contextTreeItemLessThan);
-}
 
-ContextTreeModel::ContextTreeModel(QObject *parent) : QAbstractItemModel(parent) { root = new ContextTreeItem(); }
+Model::Model(QObject *parent) :
+        QAbstractItemModel(parent),
+        root_(new Item("Elements", nullptr)) {}
 
-ContextTreeModel::~ContextTreeModel() { delete root; }
+Model::~Model() {}
 
-void ContextTreeModel::loadData(Context *ctx)
+void Model::loadContext(Context *ctx)
 {
     if (!ctx)
         return;
+    ctx_ = ctx;
 
     beginResetModel();
 
-    delete root;
+    // Currently we lack an API to get a proper hierarchy of bels/pip/wires
-    root = new ContextTreeItem();
+    // cross-arch. So we only do this for ICE40 by querying the ChipDB
-
+    // directly.
-    for (int i = 0; i < 6; i++)
+    // TODO(q3k): once AnyId and the tree API land in Arch, move this over.
-        nameToItem[i].clear();
+#ifdef ARCH_ICE40
-
+    {
-    IdString none;
+        std::map<std::pair<int, int>, std::vector<BelId>> belMap;
-
+        for (auto bel : ctx->getBels()) {
-    ContextTreeItem *bels_root = new ContextTreeItem("Bels");
+            auto loc = ctx->getBelLocation(bel);
-    root->addChild(bels_root);
+            belMap[std::pair<int, int>(loc.x, loc.y)].push_back(bel);
-    QMap<QString, ContextTreeItem *> bel_items;
-
-    // Add bels to tree
-    for (auto bel : ctx->getBels()) {
-        IdString id = ctx->getBelName(bel);
-        QStringList items = QString(id.c_str(ctx)).split("/");
-        QString name;
-        ContextTreeItem *parent = bels_root;
-        for (int i = 0; i < items.size(); i++) {
-            if (!name.isEmpty())
-                name += "/";
-            name += items.at(i);
-            if (!bel_items.contains(name)) {
-                if (i == items.size() - 1) {
-                    ContextTreeItem *item = new ContextTreeItem(id, ElementType::BEL, items.at(i));
-                    parent->addChild(item);
-                    nameToItem[0].insert(name, item);
-                } else {
-                    ContextTreeItem *item = new ContextTreeItem(none, ElementType::NONE, items.at(i));
-                    parent->addChild(item);
-                    bel_items.insert(name, item);
-                }
-            }
-            parent = bel_items[name];
         }
-    }    
+        auto belGetter = [](Context *ctx, BelId id) { return ctx->getBelName(id); };
-    bels_root->sort();
+        bel_root_ = std::unique_ptr<BelXYRoot>(new BelXYRoot(ctx, "Bels", root_.get(), belMap, belGetter, ElementType::BEL));
-    
-    ContextTreeItem *wire_root = new ContextTreeItem("Wires");
-    root->addChild(wire_root);
-    QMap<QString, ContextTreeItem *> wire_items;
 
-    // Add wires to tree
+        std::map<std::pair<int, int>, std::vector<WireId>> wireMap;
-    for (auto wire : ctx->getWires()) {
+        for (int i = 0; i < ctx->chip_info->num_wires; i++) {
-        auto id = ctx->getWireName(wire);
+            const auto wire = &ctx->chip_info->wire_data[i];
-        QStringList items = QString(id.c_str(ctx)).split("/");
+            WireId wireid;
-        QString name;
+            wireid.index = i;
-        ContextTreeItem *parent = wire_root;
+            wireMap[std::pair<int, int>(wire->x, wire->y)].push_back(wireid);
-        for (int i = 0; i < items.size(); i++) {
-            if (!name.isEmpty())
-                name += "/";
-            name += items.at(i);
-            if (!wire_items.contains(name)) {
-                if (i == items.size() - 1) {
-                    ContextTreeItem *item = new ContextTreeItem(id, ElementType::WIRE, items.at(i));
-                    parent->addChild(item);
-                    nameToItem[1].insert(name, item);
-                } else {
-                    ContextTreeItem *item = new ContextTreeItem(none, ElementType::NONE, items.at(i));
-                    parent->addChild(item);
-                    wire_items.insert(name, item);
-                }
-            }
-            parent = wire_items[name];
         }
-    }
+        auto wireGetter = [](Context *ctx, WireId id) { return ctx->getWireName(id); };
-    wire_root->sort();
+        wire_root_ = std::unique_ptr<WireXYRoot>(new WireXYRoot(ctx, "Wires", root_.get(), wireMap, wireGetter, ElementType::WIRE));
 
-    ContextTreeItem *pip_root = new ContextTreeItem("Pips");
+        std::map<std::pair<int, int>, std::vector<PipId>> pipMap;
-    root->addChild(pip_root);
+        for (int i = 0; i < ctx->chip_info->num_pips; i++) {
-    QMap<QString, ContextTreeItem *> pip_items;
+            const auto pip = &ctx->chip_info->pip_data[i];
-
+            PipId pipid;
-    // Add pips to tree
+            pipid.index = i;
-#ifndef ARCH_ECP5
+            pipMap[std::pair<int, int>(pip->x, pip->y)].push_back(pipid);
-    for (auto pip : ctx->getPips()) {
-        auto id = ctx->getPipName(pip);
-        QStringList items = QString(id.c_str(ctx)).split("/");
-        QString name;
-        ContextTreeItem *parent = pip_root;
-        for (int i = 0; i < items.size(); i++) {
-            if (!name.isEmpty())
-                name += "/";
-            name += items.at(i);
-            if (!pip_items.contains(name)) {
-                if (i == items.size() - 1) {
-                    ContextTreeItem *item = new ContextTreeItem(id, ElementType::PIP, items.at(i));
-                    parent->addChild(item);
-                    nameToItem[2].insert(name, item);
-                } else {
-                    ContextTreeItem *item = new ContextTreeItem(none, ElementType::NONE, items.at(i));
-                    parent->addChild(item);
-                    pip_items.insert(name, item);
-                }
-            }
-            parent = pip_items[name];
         }
+        auto pipGetter = [](Context *ctx, PipId id) { return ctx->getPipName(id); };
+        pip_root_ = std::unique_ptr<PipXYRoot>(new PipXYRoot(ctx, "Pips", root_.get(), pipMap, pipGetter, ElementType::PIP));
     }
 #endif
-    pip_root->sort();
 
-    nets_root = new ContextTreeItem("Nets");
+    cell_root_ = std::unique_ptr<IdStringList>(new IdStringList(QString("Cells"), root_.get(), ElementType::CELL));
-    root->addChild(nets_root);
+    net_root_ = std::unique_ptr<IdStringList>(new IdStringList(QString("Nets"), root_.get(), ElementType::NET));
-
-    cells_root = new ContextTreeItem("Cells");
-    root->addChild(cells_root);
 
     endResetModel();
+
+    updateCellsNets(ctx);
 }
 
-void ContextTreeModel::updateData(Context *ctx)
+void Model::updateCellsNets(Context *ctx)
 {
     if (!ctx)
         return;
 
     beginResetModel();
 
-    //QModelIndex nets_index = indexFromNode(nets_root);
+    std::vector<IdString> cells;
-    // Remove nets not existing any more
+    for (auto &pair : ctx->cells) {
-    QMap<QString, ContextTreeItem *>::iterator i = nameToItem[3].begin();
+        cells.push_back(pair.first);
-    while (i != nameToItem[3].end()) {
-        QMap<QString, ContextTreeItem *>::iterator prev = i;
-        ++i;
-        if (ctx->nets.find(ctx->id(prev.key().toStdString())) == ctx->nets.end()) {
-            //int pos = prev.value()->parent()->indexOf(prev.value());
-            //beginRemoveRows(nets_index, pos, pos);
-            delete prev.value();
-            nameToItem[3].erase(prev);
-            //endRemoveRows();
-        }
-    }
-    // Add nets to tree
-    for (auto &item : ctx->nets) {
-        auto id = item.first;
-        QString name = QString(id.c_str(ctx));
-        if (!nameToItem[3].contains(name)) {
-            //beginInsertRows(nets_index, nets_root->count() + 1, nets_root->count() + 1);
-            ContextTreeItem *newItem = new ContextTreeItem(id, ElementType::NET, name);
-            nets_root->addChild(newItem);
-            nameToItem[3].insert(name, newItem);
-            //endInsertRows();
-        }
     }
+    cell_root_->updateElements(ctx, cells);
 
-    nets_root->sort();
+    std::vector<IdString> nets;
-
+    for (auto &pair : ctx->nets) {
-    //QModelIndex cell_index = indexFromNode(cells_root);
+        nets.push_back(pair.first);
-    // Remove cells not existing any more
-    i = nameToItem[4].begin();
-    while (i != nameToItem[4].end()) {
-        QMap<QString, ContextTreeItem *>::iterator prev = i;
-        ++i;
-        if (ctx->cells.find(ctx->id(prev.key().toStdString())) == ctx->cells.end()) {
-            //int pos = prev.value()->parent()->indexOf(prev.value());
-            //beginRemoveRows(cell_index, pos, pos);
-            delete prev.value();
-            nameToItem[4].erase(prev);
-            //endRemoveRows();
-        }
     }
-    // Add cells to tree
+    net_root_->updateElements(ctx, nets);
-    for (auto &item : ctx->cells) {
-        auto id = item.first;
-        QString name = QString(id.c_str(ctx));
-        if (!nameToItem[4].contains(name)) {
-            //beginInsertRows(cell_index, cells_root->count() + 1, cells_root->count() + 1);
-            ContextTreeItem *newItem = new ContextTreeItem(id, ElementType::CELL, name);
-            cells_root->addChild(newItem);
-            nameToItem[4].insert(name, newItem);
-            //endInsertRows();
-        }
-    }
-
-    cells_root->sort();
 
     endResetModel();
 }
 
-int ContextTreeModel::rowCount(const QModelIndex &parent) const { return nodeFromIndex(parent)->count(); }
+int Model::rowCount(const QModelIndex &parent) const { return nodeFromIndex(parent)->count(); }
 
-int ContextTreeModel::columnCount(const QModelIndex &parent) const { return 1; }
+int Model::columnCount(const QModelIndex &parent) const { return 1; }
 
-QModelIndex ContextTreeModel::index(int row, int column, const QModelIndex &parent) const
+QModelIndex Model::index(int row, int column, const QModelIndex &parent) const
 {
-    ContextTreeItem *node = nodeFromIndex(parent);
+    Item *node = nodeFromIndex(parent);
     if (row >= node->count())
         return QModelIndex();
-    return createIndex(row, column, node->at(row));
+
+    return createIndex(row, column, node->child(row));
 }
 
-QModelIndex ContextTreeModel::parent(const QModelIndex &child) const
+QModelIndex Model::parent(const QModelIndex &child) const
 {
-    ContextTreeItem *parent = nodeFromIndex(child)->parent();
+    Item *parent = nodeFromIndex(child)->parent();
-    if (parent == root)
+    if (parent == root_.get())
         return QModelIndex();
-    ContextTreeItem *node = parent->parent();
+    Item *node = parent->parent();
     return createIndex(node->indexOf(parent), 0, parent);
 }
 
-QVariant ContextTreeModel::data(const QModelIndex &index, int role) const
+QVariant Model::data(const QModelIndex &index, int role) const
 {
     if (index.column() != 0)
         return QVariant();
     if (role != Qt::DisplayRole)
         return QVariant();
-    ContextTreeItem *node = nodeFromIndex(index);
+    Item *node = nodeFromIndex(index);
     return node->name();
 }
 
-QVariant ContextTreeModel::headerData(int section, Qt::Orientation orientation, int role) const
+QVariant Model::headerData(int section, Qt::Orientation orientation, int role) const
 {
     Q_UNUSED(section);
     if (orientation == Qt::Horizontal && role == Qt::DisplayRole)
@@ -285,62 +271,53 @@ QVariant ContextTreeModel::headerData(int section, Qt::Orientation orientation,
     return QVariant();
 }
 
-ContextTreeItem *ContextTreeModel::nodeFromIndex(const QModelIndex &idx) const
+Item *Model::nodeFromIndex(const QModelIndex &idx) const
 {
     if (idx.isValid())
-        return (ContextTreeItem *)idx.internalPointer();
+        return (Item *)idx.internalPointer();
-    return root;
+    return root_.get();
 }
 
-static int getElementIndex(ElementType type)
+Qt::ItemFlags Model::flags(const QModelIndex &index) const
 {
-    if (type == ElementType::BEL)
+    Item *node = nodeFromIndex(index);
-        return 0;
-    if (type == ElementType::WIRE)
-        return 1;
-    if (type == ElementType::PIP)
-        return 2;
-    if (type == ElementType::NET)
-        return 3;
-    if (type == ElementType::CELL)
-        return 4;
-    return -1;
-}
-
-ContextTreeItem *ContextTreeModel::nodeForIdType(const ElementType type, const QString name) const
-{
-    int index = getElementIndex(type);
-    if (type != ElementType::NONE && nameToItem[index].contains(name))
-        return nameToItem[index].value(name);
-    return nullptr;
-}
-
-QModelIndex ContextTreeModel::indexFromNode(ContextTreeItem *node)
-{
-    ContextTreeItem *parent = node->parent();
-    if (parent == root)
-        return QModelIndex();
-    return createIndex(parent->indexOf(node), 0, node);
-}
-
-Qt::ItemFlags ContextTreeModel::flags(const QModelIndex &index) const
-{
-    ContextTreeItem *node = nodeFromIndex(index);
     return Qt::ItemIsEnabled | (node->type() != ElementType::NONE ? Qt::ItemIsSelectable : Qt::NoItemFlags);
 }
 
-QList<QModelIndex> ContextTreeModel::search(QString text)
+
+void Model::fetchMore(const QModelIndex &parent)
 {
-    QList<QModelIndex> list;
+    if (ctx_ == nullptr)
-    for (int i = 0; i < 6; i++) {
+        return;
-        for (auto key : nameToItem[i].keys()) {
+
-            if (key.contains(text, Qt::CaseInsensitive)) {
+    std::lock_guard<std::mutex> lock_ui(ctx_->ui_mutex);
-                list.append(indexFromNode(nameToItem[i].value(key)));
+    std::lock_guard<std::mutex> lock(ctx_->mutex);
-                if (list.count() > 500)
+
-                    break; // limit to 500 results
+    nodeFromIndex(parent)->fetchMore();
-            }
-        }
-    }
-    return list;
 }
+
+bool Model::canFetchMore(const QModelIndex &parent) const
+{
+    return nodeFromIndex(parent)->canFetchMore();
+}
+
+QList<QModelIndex> Model::search(QString text)
+{
+    const int limit = 500;
+    QList<Item*> list;
+    cell_root_->search(list, text, limit);
+    net_root_->search(list, text, limit);
+    bel_root_->search(list, text, limit);
+    wire_root_->search(list, text, limit);
+    pip_root_->search(list, text, limit);
+
+    QList<QModelIndex> res;
+    for (auto i : list) {
+        res.push_back(indexFromNode(i));
+    }
+    return res;
+}
+
+}; // namespace TreeModel
+
 NEXTPNR_NAMESPACE_END

gui/treemodel.h

@@ -2,6 +2,7 @@
  *  nextpnr -- Next Generation Place and Route
  *
  *  Copyright (C) 2018  Miodrag Milanovic <miodrag@symbioticeda.com>
+ *  Copyright (C) 2018  Serge Bazanski <q3k@symbioticeda.com>
  *
  *  Permission to use, copy, modify, and/or distribute this software for any
  *  purpose with or without fee is hereby granted, provided that the above
@@ -21,6 +22,8 @@
 #define TREEMODEL_H
 
 #include <QAbstractItemModel>
+#include <boost/optional.hpp>
+
 #include "nextpnr.h"
 
 NEXTPNR_NAMESPACE_BEGIN
@@ -36,43 +39,365 @@ enum class ElementType
     GROUP
 };
 
-class ContextTreeItem
+namespace TreeModel {
-{
-  public:
-    ContextTreeItem();
-    ContextTreeItem(QString name);
-    ContextTreeItem(IdString id, ElementType type, QString name);
-    ~ContextTreeItem();
 
-    void addChild(ContextTreeItem *item);
+// Item is a leaf or non-leaf item in the TreeModel hierarchy. It does not
-    int indexOf(ContextTreeItem *n) const { return children.indexOf(n); }
+// manage any memory.
-    ContextTreeItem *at(int idx) const { return children.at(idx); }
+// It has a list of children, and when created it registers itself as a child
-    int count() const { return children.count(); }
+// of its parent.
-    ContextTreeItem *parent() const { return parentNode; }
+// It has some PNR-specific members, like type (if any), idstring (if ay).
-    IdString id() const { return itemId; }
+// They should be overwritten by deriving classes to make them relate to an
-    ElementType type() const { return itemType; }
+// object somewhere in the arch universe.
-    QString name() const { return itemName; }
+// It also has provisions for lazy loading of data, via the canFetchMore and
-    void sort();
+// fetchMore methods.
-  private:
+class Item
-    ContextTreeItem *parentNode;
+{
-    QList<ContextTreeItem *> children;
+  protected:
-    IdString itemId;
+    // Human-friendly name of this item.
-    ElementType itemType;
+    QString name_;
-    QString itemName;
+    // Parent or nullptr if root.
+    Item *parent_;
+    // Children that are loaded into memory.
+    QList<Item *> children_;
+
+    void addChild(Item *child)
+    {
+        children_.append(child);
+    }
+
+  public:
+    Item(QString name, Item *parent) :
+            name_(name), parent_(parent)
+    {
+        // Register in parent if exists.
+        if (parent_ != nullptr) {
+            parent_->addChild(this);
+        }
+    };
+
+    // Number of children.
+    int count() const { return children_.count(); }
+
+    // Name getter.
+    QString name() const { return name_; }
+
+    // Child getter.
+    Item *child(int index) { return children_.at(index); }
+
+    // Parent getter.
+    const Item *parent() const { return parent_; }
+    Item *parent() { return parent_; }
+
+    // indexOf gets index of child in children array.
+    int indexOf(const Item *child) const
+    { 
+        // Dropping the const for indexOf to work.
+        return children_.indexOf((Item *)child, 0);
+    }
+    int indexOf(Item *child) { return children_.indexOf(child, 0); }
+
+    // Arch id and type that correspond to this element.
+    virtual IdString id() const { return IdString(); }
+    virtual ElementType type() const { return ElementType::NONE; }
+
+    // Lazy loading methods.
+    virtual bool canFetchMore() const { return false; }
+    virtual void fetchMore() {}
+
+    ~Item() {}
 };
 
-class ContextTreeModel : public QAbstractItemModel
+// IdString is an Item that corresponds to a real element in Arch.
+class IdStringItem : public Item
+{
+  private:
+    IdString id_;
+    ElementType type_;
+
+  public:
+    IdStringItem(Context *ctx, IdString str, Item *parent, ElementType type) :
+            Item(QString(str.c_str(ctx)), parent), id_(str), type_(type) {}
+
+    virtual IdString id() const override
+    {
+        return id_;
+    }
+
+    virtual ElementType type() const override
+    {
+       return type_;
+    }
+};
+
+// IdString list is a static list of IdStrings which can be set/updates from
+// a vector of IdStrings. It will render each IdStrings as a child, with the
+// list sorted in a smart way. 
+class IdStringList : public Item
+{
+  private:
+    // Children that we manage the memory for, stored for quick lookup from
+    // IdString to child.
+    std::unordered_map<IdString, std::unique_ptr<IdStringItem>> managed_;
+    // Type of children that the list creates.
+    ElementType child_type_;
+
+  public:
+    // Create an IdStringList at given partent that will contain elements of
+    // the given type.
+    IdStringList(QString name, Item *parent, ElementType type) :
+            Item(name, parent), child_type_(type) {}
+
+    // Split a name into alpha/non-alpha parts, which is then used for sorting
+    // of children.
+    static std::vector<QString> alphaNumSplit(const QString &str);
+
+    // getById finds a child for the given IdString.
+    IdStringItem *getById(IdString id) const
+    {
+        return managed_.at(id).get();
+    }
+
+    // (Re-)create children from a list of IdStrings.
+    void updateElements(Context *ctx, std::vector<IdString> elements);
+
+    // Find children that contain the given text.
+    void search(QList<Item*> &results, QString text, int limit);
+};
+
+
+// ElementList is a dynamic list of ElementT (BelId,WireId,...) that are
+// automatically generated based on an overall map of elements.
+// ElementList is emitted from ElementXYRoot, and contains the actual
+// Bels/Wires/Pips underneath it.
+template <typename ElementT>
+class ElementList : public Item
 {
   public:
-    ContextTreeModel(QObject *parent = nullptr);
+    // A map from tile (X,Y) to list of ElementTs in that tile.
-    ~ContextTreeModel();
+    using ElementMap = std::map<std::pair<int, int>, std::vector<ElementT>>;
+    // A method that converts an ElementT to an IdString.
+    using ElementGetter = std::function<IdString(Context *, ElementT)>;
+
+  private:
+    Context *ctx_;
+    // ElementMap given to use by our constructor.
+    const ElementMap *map_;
+    // The X, Y that this list handles.
+    int x_, y_;
+    ElementGetter getter_;
+    // Children that we manage the memory for, stored for quick lookup from
+    // IdString to child.
+    std::unordered_map<IdString, std::unique_ptr<Item>> managed_;
+    // Type of children that he list creates.
+    ElementType child_type_;
+
+    // Gets elements that this list should create from the map. This pointer is
+    // short-lived (as it will change when the map mutates.
+    const std::vector<ElementT> *elements() const
+    {
+        return &map_->at(std::make_pair(x_, y_));
+    }
+
+  public:
+    ElementList(Context *ctx, QString name, Item *parent, ElementMap *map, int x, int y, ElementGetter getter, ElementType type) :
+            Item(name, parent), ctx_(ctx), map_(map), x_(x), y_(y), getter_(getter), child_type_(type)
+    {
+    }
+
+    // Lazy loading of elements.
+
+    virtual bool canFetchMore() const override
+    {
+        return (size_t)children_.size() < elements()->size();
+    }
+
+    void fetchMore(int count)
+    {
+        size_t start = children_.size();
+        size_t end = std::min(start + count, elements()->size());
+        for (size_t i = start; i < end; i++) {
+            auto idstring = getter_(ctx_, elements()->at(i));
+            QString name(idstring.c_str(ctx_));
+
+            // Remove X.../Y.../ prefix
+            QString prefix = QString("X%1/Y%2/").arg(x_).arg(y_);
+            if (name.startsWith(prefix))
+                name.remove(0, prefix.size());
+
+            auto item = new IdStringItem(ctx_, idstring, this, child_type_);
+            managed_[idstring] = std::move(std::unique_ptr<Item>(item));
+        }
+    }
+
+    virtual void fetchMore() override
+    {
+         fetchMore(100);
+    }
+
+    // getById finds a child for the given IdString.
+    boost::optional<Item*> getById(IdString id)
+    {
+        // Search requires us to load all our elements...
+        while (canFetchMore()) fetchMore();
+
+        auto res = managed_.find(id);
+        if (res != managed_.end()) {
+            return res->second.get();
+        }
+        return boost::none;
+    }
+
+    // Find children that contain the given text.
+    void search(QList<Item*> &results, QString text, int limit)
+    {
+        // Last chance to bail out from loading entire tree into memory.
+        if (limit != -1 && results.size() > limit)
+            return;
+
+        // Search requires us to load all our elements...
+        while (canFetchMore()) fetchMore();
+
+        for (const auto &child : children_) {
+            if (limit != -1 && results.size() > limit)
+                return;
+            if (child->name().contains(text))
+                results.push_back(child);
+        }
+    }
+};
+
+// ElementXYRoot is the root of an ElementT multi-level lazy loading list.
+// It can take any of {BelId,WireId,PipId} and create a tree that
+// hierarchizes them by X and Y tile positions, when given a map from X,Y to
+// list of ElementTs in that tile.
+template <typename ElementT>
+class ElementXYRoot : public Item
+{
+  public:
+    // A map from tile (X,Y) to list of ElementTs in that tile.
+    using ElementMap = std::map<std::pair<int, int>, std::vector<ElementT>>;
+    // A method that converts an ElementT to an IdString.
+    using ElementGetter = std::function<IdString(Context *, ElementT)>;
+
+
+  private:
+    Context *ctx_;
+    // X-index children that we manage the memory for.
+    std::vector<std::unique_ptr<Item>> managed_labels_;
+    // Y-index children (ElementLists) that we manage the memory for.
+    std::vector<std::unique_ptr<ElementList<ElementT>>> managed_lists_;
+    // Source of truth for elements to display.
+    ElementMap map_;
+    ElementGetter getter_;
+    // Type of children that he list creates in X->Y->...
+    ElementType child_type_;
+
+  public:
+    ElementXYRoot(Context *ctx, QString name, Item *parent, ElementMap map, ElementGetter getter, ElementType type) :
+            Item(name, parent), ctx_(ctx), map_(map), getter_(getter), child_type_(type)
+    {
+        // Create all X and Y label Items/ElementLists.
+
+        // Y coordinates at which an element exists for a given X - taken out
+        // of loop to limit heap allocation/deallocation.
+        std::vector<int> y_present;
+
+        for (int i = 0; i < ctx->getGridDimX(); i++) {
+            y_present.clear();
+            // First find all the elements in all Y coordinates in this X.
+            for (int j = 0; j < ctx->getGridDimY(); j++) {
+                if (map_.count(std::make_pair(i, j)) == 0)
+                    continue;
+                y_present.push_back(j);
+            }
+            // No elements in any X coordinate? Do not add X tree item.
+            if (y_present.size() == 0)
+                continue;
+
+            // Create X list Item.
+            auto item = new Item(QString("X%1").arg(i), this);
+            managed_labels_.push_back(std::move(std::unique_ptr<Item>(item)));
+
+            for (auto j : y_present) {
+                // Create Y list ElementList.
+                auto item2 = new ElementList<ElementT>(ctx_, QString("Y%1").arg(j), item, &map_, i, j, getter_, child_type_);
+                // Pre-populate list with one element, other Qt will never ask for more.
+                item2->fetchMore(1);
+                managed_lists_.push_back(std::move(std::unique_ptr<ElementList<ElementT>>(item2)));
+            }
+        }
+    }
+
+    // getById finds a child for the given IdString.
+    boost::optional<Item*> getById(IdString id)
+    {
+        // For now, scan linearly all ElementLists.
+        // TODO(q3k) fix this once we have tree API from arch
+        for (auto &l : managed_lists_) {
+            auto res = l->getById(id);
+            if (res) {
+                return res;
+            }
+        }
+        return boost::none;
+    }
+
+    // Find children that contain the given text.
+    void search(QList<Item*> &results, QString text, int limit)
+    {
+        for (auto &l : managed_lists_) {
+            if (limit != -1 && results.size() > limit)
+                return;
+            l->search(results, text, limit);
+        }
+    }
+};
+
+class Model : public QAbstractItemModel
+{
+  private:
+    Context *ctx_ = nullptr;
+
+  public:
+    using BelXYRoot = ElementXYRoot<BelId>;
+    using WireXYRoot = ElementXYRoot<WireId>;
+    using PipXYRoot = ElementXYRoot<PipId>;
+
+    Model(QObject *parent = nullptr);
+    ~Model();
+
+    void loadContext(Context *ctx);
+    void updateCellsNets(Context *ctx);
+    Item *nodeFromIndex(const QModelIndex &idx) const;
+    QModelIndex indexFromNode(Item *node)
+    {
+        const Item *parent = node->parent();
+        if (parent == nullptr)
+            return QModelIndex();
+        
+        return createIndex(parent->indexOf(node), 0, node);
+    }
 
-    void loadData(Context *ctx);
-    void updateData(Context *ctx);
-    ContextTreeItem *nodeFromIndex(const QModelIndex &idx) const;
-    QModelIndex indexFromNode(ContextTreeItem *node);
-    ContextTreeItem *nodeForIdType(const ElementType type, const QString name) const;
     QList<QModelIndex> search(QString text);
+
+    boost::optional<Item*> nodeForIdType(ElementType type, IdString id) const
+    {
+        switch (type) {
+        case ElementType::BEL:
+            return bel_root_->getById(id);
+        case ElementType::WIRE:
+            return wire_root_->getById(id);
+        case ElementType::PIP:
+            return pip_root_->getById(id);
+        case ElementType::CELL:
+            return cell_root_->getById(id);
+        case ElementType::NET:
+            return net_root_->getById(id);
+        default:
+            return boost::none;
+        }
+    }
+ 
     // Override QAbstractItemModel methods
     int rowCount(const QModelIndex &parent = QModelIndex()) const Q_DECL_OVERRIDE;
     int columnCount(const QModelIndex &parent = QModelIndex()) const Q_DECL_OVERRIDE;
@@ -81,14 +406,22 @@ class ContextTreeModel : public QAbstractItemModel
     QVariant data(const QModelIndex &index, int role = Qt::DisplayRole) const Q_DECL_OVERRIDE;
     QVariant headerData(int section, Qt::Orientation orientation, int role) const Q_DECL_OVERRIDE;
     Qt::ItemFlags flags(const QModelIndex &index) const Q_DECL_OVERRIDE;
+    void fetchMore(const QModelIndex &parent) Q_DECL_OVERRIDE;
+    bool canFetchMore(const QModelIndex &parent) const Q_DECL_OVERRIDE;
 
   private:
-    ContextTreeItem *root;
+
-    QMap<QString, ContextTreeItem *> nameToItem[6];
+    // Tree elements that we manage the memory for.
-    ContextTreeItem *nets_root;
+    std::unique_ptr<Item> root_;
-    ContextTreeItem *cells_root;
+    std::unique_ptr<BelXYRoot> bel_root_;
+    std::unique_ptr<WireXYRoot> wire_root_;
+    std::unique_ptr<PipXYRoot> pip_root_;
+    std::unique_ptr<IdStringList> cell_root_;
+    std::unique_ptr<IdStringList> net_root_;
 };
 
+}; // namespace TreeModel
+
 NEXTPNR_NAMESPACE_END
 
 #endif // TREEMODEL_H