zotero/user.sql
Dan Stillman ab13c3980a Fulltext search support
There are currently two types of fulltext searching: an SQL-based word index and a file scanner. They each have their advantages and drawbacks.

The word index is very fast to search and is currently used for the find-as-you-type quicksearch. However, indexing files takes some time, so we should probably offer a preference to turn it off ("Index attachment content for quicksearch" or something). There's also an issue with Chinese characters (which are indexed by character rather than word, since there are no spaces to go by, so a search for a word with common characters could produce erroneous results). The quicksearch doesn't use a left-bound index (since that would probably upset German speakers searching for "musik" in "nachtmusik," though I don't know for sure how they think of words) but still seems pretty fast.

* Note: There will be a potentially long delay when you start Firefox with this revision as it builds a fulltext word index of your existing items. We obviously need a notification/option for this. *

The file scanner, used in the Attachment Content condition of the search dialog, offers phrase searching as well as regex support (both case-sensitive and not, and defaulting to multiline). It doesn't require an index, though it should probably be optimized to use the word index, if available, for narrowing the results when not in regex mode. (It does only scan files that pass all the other search conditions, which speeds it up considerably for multi-condition searches, and skips non-text files unless instructed otherwise, but it's still relatively slow.)

Both convert HTML to text before searching (with the exception of the binary file scanning mode).

There are some issues with which files get indexed and which don't that we can't do much about and that will probably confuse users immensely. Dan C. suggested some sort of indicator (say, a green dot) to show which files are indexed.

Also added (very ugly) charset detection (anybody want to figure out getCharsetFromString(str)?), a setTimeout() replacement in the XPCOM service, an arrayToHash() method, and a new header to timedtextarea.xml, since it's really not copyright CHNM (it's really just a few lines off from the toolkit timed-textbox binding--I tried to change it to extend timed-textbox and just ignore Return keypress events so that we didn't need to duplicate the Mozilla code, but timed-textbox's reliance on html:input instead of html:textarea made things rather difficult).

To do:

- Pref/buttons to disable/clear/rebuild fulltext index
- Hidden prefs to set maximum file size to index/scan
- Don't index words of fewer than 3 non-Asian characters
- MRU cache for saved searches
- Use word index if available to narrow search scope of fulltext scanner
- Cache attachment info methods
- Show content excerpt in search results (at least in advanced search window, when it exists)
- Notification window (a la scraping) to show when indexing
- Indicator of indexed status
- Context menu option to index
- Indicator that a file scanning search is in progress, if possible
- Find other ways to make it index the NYT front page in under 10 seconds
- Probably fix lots of bugs, which you will likely start telling me about...now.
2006-09-21 00:10:29 +00:00

191 lines
5.5 KiB
SQL

-- 3
-- This file creates tables containing user-specific data -- any changes
-- to existing tables made here must be mirrored in transition steps in
-- schema.js::_migrateSchema() -- however, new tables can be added by simply
-- adding a CREATE TABLE IF NOT EXISTS statement and incrementing the version
-- number above
CREATE TABLE IF NOT EXISTS version (
schema TEXT PRIMARY KEY,
version INT NOT NULL
);
CREATE INDEX IF NOT EXISTS schema ON version(schema);
-- Show or hide pre-mapped fields for system item types
CREATE TABLE IF NOT EXISTS userFieldMask (
itemTypeID INT,
fieldID INT,
hide INT,
PRIMARY KEY (itemTypeID, fieldID),
FOREIGN KEY (itemTypeID, fieldID) REFERENCES itemTypeFields(itemTypeID, fieldID)
);
-- User-defined item types -- itemTypeIDs must be >= 1000
CREATE TABLE IF NOT EXISTS userItemTypes (
itemTypeID INT,
typeName TEXT,
templateItemTypeID INT,
PRIMARY KEY (itemTypeID)
);
-- User-defined fields
CREATE TABLE IF NOT EXISTS userFields (
userFieldID INT,
fieldName TEXT,
PRIMARY KEY (userFieldID)
);
-- Map custom fields to system and custom item types
CREATE TABLE IF NOT EXISTS userItemTypeFields (
itemTypeID INT,
userFieldID INT,
orderIndex INT,
PRIMARY KEY (itemTypeID, userFieldID),
FOREIGN KEY (userFieldID) REFERENCES userFields(userFieldID)
);
-- The foundational table; every item collected has a unique record here
CREATE TABLE IF NOT EXISTS items (
itemID INTEGER PRIMARY KEY,
itemTypeID INT,
title TEXT,
dateAdded DATETIME DEFAULT CURRENT_TIMESTAMP,
dateModified DATETIME DEFAULT CURRENT_TIMESTAMP
);
-- Type-specific data for individual items
CREATE TABLE IF NOT EXISTS itemData (
itemID INT,
fieldID INT,
value,
PRIMARY KEY (itemID, fieldID),
FOREIGN KEY (itemID) REFERENCES items(itemID),
FOREIGN KEY (fieldID) REFERENCES fields(fieldID)
);
CREATE INDEX IF NOT EXISTS value ON itemData(value);
-- Note data for note items
CREATE TABLE IF NOT EXISTS itemNotes (
itemID INT,
sourceItemID INT,
note TEXT,
PRIMARY KEY (itemID),
FOREIGN KEY (itemID) REFERENCES items(itemID),
FOREIGN KEY (sourceItemID) REFERENCES items(itemID)
);
CREATE INDEX IF NOT EXISTS itemNotes_sourceItemID ON itemNotes(sourceItemID);
-- Metadata for attachment items
CREATE TABLE IF NOT EXISTS itemAttachments (
itemID INT,
sourceItemID INT,
linkMode INT,
mimeType TEXT,
charsetID INT,
path TEXT,
originalPath TEXT,
PRIMARY KEY (itemID),
FOREIGN KEY (itemID) REFERENCES items(itemID),
FOREIGN KEY (sourceItemID) REFERENCES items(sourceItemID)
);
CREATE INDEX IF NOT EXISTS itemAttachments_sourceItemID ON itemAttachments(sourceItemID);
CREATE INDEX IF NOT EXISTS itemAttachments_mimeType ON itemAttachments(mimeType);
-- Individual entries for each tag
CREATE TABLE IF NOT EXISTS tags (
tagID INT,
tag TEXT UNIQUE,
PRIMARY KEY (tagID)
);
-- Associates items with keywords
CREATE TABLE IF NOT EXISTS itemTags (
itemID INT,
tagID INT,
PRIMARY KEY (itemID, tagID),
FOREIGN KEY (itemID) REFERENCES items(itemID),
FOREIGN KEY (tagID) REFERENCES tags(tagID)
);
CREATE INDEX IF NOT EXISTS itemTags_tagID ON itemTags(tagID);
CREATE TABLE IF NOT EXISTS itemSeeAlso (
itemID INT,
linkedItemID INT,
PRIMARY KEY (itemID, linkedItemID),
FOREIGN KEY (itemID) REFERENCES items(itemID),
FOREIGN KEY (linkedItemID) REFERENCES items(itemID)
);
CREATE INDEX IF NOT EXISTS itemSeeAlso_linkedItemID ON itemSeeAlso(linkedItemID);
-- Names of each individual "creator" (inc. authors, editors, etc.)
CREATE TABLE IF NOT EXISTS creators (
creatorID INT,
firstName TEXT,
lastName TEXT,
isInstitution INT,
PRIMARY KEY (creatorID)
);
-- Associates single or multiple creators to items
CREATE TABLE IF NOT EXISTS itemCreators (
itemID INT,
creatorID INT,
creatorTypeID INT DEFAULT 1,
orderIndex INT DEFAULT 0,
PRIMARY KEY (itemID, creatorID, creatorTypeID, orderIndex),
FOREIGN KEY (itemID) REFERENCES items(itemID),
FOREIGN KEY (creatorID) REFERENCES creators(creatorID)
FOREIGN KEY (creatorTypeID) REFERENCES creatorTypes(creatorTypeID)
);
-- Collections for holding items
CREATE TABLE IF NOT EXISTS collections (
collectionID INT,
collectionName TEXT,
parentCollectionID INT,
PRIMARY KEY (collectionID),
FOREIGN KEY (parentCollectionID) REFERENCES collections(collectionID)
);
-- Associates items with the various collections they belong to
CREATE TABLE IF NOT EXISTS collectionItems (
collectionID INT,
itemID INT,
orderIndex INT DEFAULT 0,
PRIMARY KEY (collectionID, itemID),
FOREIGN KEY (collectionID) REFERENCES collections(collectionID),
FOREIGN KEY (itemID) REFERENCES items(itemID)
);
CREATE INDEX IF NOT EXISTS itemID ON collectionItems(itemID);
CREATE TABLE IF NOT EXISTS savedSearches (
savedSearchID INT,
savedSearchName TEXT,
PRIMARY KEY(savedSearchID)
);
CREATE TABLE IF NOT EXISTS savedSearchConditions (
savedSearchID INT,
searchConditionID INT,
condition TEXT,
operator TEXT,
value TEXT,
required NONE,
PRIMARY KEY(savedSearchID, searchConditionID),
FOREIGN KEY (savedSearchID) REFERENCES savedSearches(savedSearchID)
);
CREATE TABLE IF NOT EXISTS fulltextWords (
wordID INTEGER PRIMARY KEY,
word TEXT UNIQUE
);
CREATE INDEX IF NOT EXISTS fulltextWords_word ON fulltextWords(word);
CREATE TABLE IF NOT EXISTS fulltextItems (
wordID INT,
itemID INT,
PRIMARY KEY (wordID, itemID)
);
CREATE INDEX IF NOT EXISTS fulltextItems_itemID ON fulltextItems(itemID);