libscid 0.1.0
Chess applications made easy.
Loading...
Searching...
No Matches
scidbase.h
1/*
2# Copyright (C) 2014-2019 Fulvio Benini
3
4* This file is part of Scid (Shane's Chess Information Database).
5*
6* Scid is free software: you can redistribute it and/or modify
7* it under the terms of the GNU General Public License as published by
8* the Free Software Foundation.
9*
10* Scid is distributed in the hope that it will be useful,
11* but WITHOUT ANY WARRANTY; without even the implied warranty of
12* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13* GNU General Public License for more details.
14*
15* You should have received a copy of the GNU General Public License
16* along with Scid. If not, see <http://www.gnu.org/licenses/>.
17*/
18
19#ifndef SCIDBASE_H
20#define SCIDBASE_H
21
22#include "scid/core/game.h"
23#include "scid/core/game_result.h"
24#include "scid/core/fullmove.h"
25#include "scid/core/board.h"
26#include "scid/database/game_id.h"
27#include "scid/database/game_info.h"
28#include "scid/database/hfilter.h"
29#include "scid/database/index.h"
30#include "scid/database/namebase.h"
31#include "scid/database/tree.h"
32#include <array>
33#include <cassert>
34#include <memory>
35#include <optional>
36#include <string>
37#include <string_view>
38#include <unordered_map>
39#include <vector>
40
41namespace scid::core {
42class Position;
43}
44
45namespace scid::database {
46
47class ByteBuffer;
48class GameView;
49class Progress;
50class SortCache;
51
52// Pattern filter for material searches.
53// It can specify, for example, a white pawn on the f-file, or a black bishop
54// on f2 and white king on e1.
55struct patternT {
56 scid::core::pieceT pieceMatch;
57 scid::core::rankT rankMatch;
58 scid::core::fyleT fyleMatch;
59 scid::core::byte flag; // 0 means this pattern must not occur.
60};
61
62enum gameExactMatchT : int {
63 GAME_EXACT_MATCH_Exact = 0,
64 GAME_EXACT_MATCH_Pawns,
65 GAME_EXACT_MATCH_Fyles,
66 GAME_EXACT_MATCH_Material
67};
68
69struct scidBaseT {
70 struct RatingUpdateStats {
71 scid::core::uint changedRatings = 0;
72 scid::core::uint changedGames = 0;
73 };
74
75 struct Stats {
76 scid::core::uint flagCount[IndexEntry::IDX_NUM_FLAGS]; // Num of games with each
77 // flag set.
78 scid::core::dateT minDate;
79 scid::core::dateT maxDate;
80 uint64_t nYears;
81 uint64_t sumYears;
82 scid::core::uint nResults[scid::core::NUM_RESULT_TYPES];
83 scid::core::uint nRatings;
84 uint64_t sumRatings;
85 scid::core::uint minRating;
86 scid::core::uint maxRating;
87
88 Stats(const scidBaseT* dbase);
89
90 struct Eco {
91 scid::core::uint count;
92 scid::core::uint results[scid::core::NUM_RESULT_TYPES];
93
94 Eco();
95 };
96 const Eco* getEcoStats(const char* ecoStr) const;
97
98 private:
99 Eco ecoEmpty_;
100 Eco ecoValid_;
101 Eco ecoStats_[(1 + (1 << 16) / 131) * 27];
102 Eco ecoGroup1_[(1 + (1 << 16) / 131) / 100];
103 Eco ecoGroup2_[(1 + (1 << 16) / 131) / 10];
104 Eco ecoGroup3_[(1 + (1 << 16) / 131)];
105 };
106
107 scidBaseT();
108 ~scidBaseT();
109
110 scid::core::errorT open(std::string_view dbType, fileModeT fMode, const char* filename,
111 const Progress& progress = {});
112
113 void Close();
114
115 std::string getFileName() const;
116 bool isOpen() const { return inUse; }
117 bool isReadOnly() const { return fileMode_ == FMODE_ReadOnly; }
118 gamenumT numGames() const { return idx->GetNumGames(); }
119
122 std::vector<std::pair<const char*, std::string>> getExtraInfo() const;
123
125 scid::core::errorT setExtraInfo(const char* tagname, const char* new_value);
126
127 const IndexEntry* getIndexEntry(gamenumT g) const {
128 assert(g < numGames());
129 return idx->GetEntry(g);
130 }
131 const IndexEntry* getIndexEntry_bounds(gamenumT g) const {
132 static_assert(std::is_unsigned_v<gamenumT>);
133 return g < numGames() ? getIndexEntry(g) : nullptr;
134 }
135 GameInfo gameInfo(gamenumT g) const;
136 std::optional<GameInfo> gameInfoBounds(gamenumT g) const {
137 static_assert(std::is_unsigned_v<gamenumT>);
138 return g < numGames() ? std::optional<GameInfo>{gameInfo(g)}
139 : std::nullopt;
140 }
141 scid::core::errorT updateGameInfo(gamenumT g, const GameInfoUpdate& update);
142 TagRoster tagRoster(gamenumT gnum) const {
143 return tagRoster(*getIndexEntry(gnum));
144 }
145 TagRoster tagRoster(IndexEntry const& ie) const {
146 return TagRoster::make(ie, *nb_);
147 }
148
149 const NameBase* getNameBase() const { return nb_; }
150
152 scid::core::ratingT peakElo(idNumberT playerID) const {
153 if (peakEloCache_.empty()) {
154 for (gamenumT gnum = 0, n = numGames(); gnum < n; gnum++) {
155 IndexEntry const& ie = *getIndexEntry(gnum);
156 auto updateMax = [&](auto id, auto elo) {
157 auto& max_value = peakEloCache_[id];
158 max_value = std::max(max_value, elo);
159 };
160 updateMax(ie.GetWhite(), ie.GetWhiteElo());
161 updateMax(ie.GetBlack(), ie.GetBlackElo());
162 }
163 }
164 return peakEloCache_[playerID];
165 }
166
167 scid::core::errorT loadGame(const IndexEntry& ie, scid::core::Game& dest,
168 char* scidFlags, std::size_t scidFlagsLen) const;
169 scid::core::errorT loadGame(gamenumT gNum, scid::core::Game& dest,
170 char* scidFlags, std::size_t scidFlagsLen) const;
171 scid::core::errorT loadGameMovesOnly(gamenumT gNum,
172 scid::core::Game& dest) const;
173 scid::core::errorT loadGameMovesOnly(const IndexEntry& ie,
174 scid::core::Game& dest) const;
175 scid::core::errorT gameTags(
176 gamenumT gNum,
177 std::vector<std::pair<std::string, std::string>>& dest) const;
178 scid::core::errorT loadStandardTags(gamenumT gNum,
179 scid::core::Game& dest,
180 char* scidFlags,
181 std::size_t scidFlagsLen) const;
182 scid::core::errorT gameTags(
183 const IndexEntry& ie,
184 std::vector<std::pair<std::string, std::string>>& dest) const;
185 std::vector<scid::core::FullMove> mainlineMoves(
186 gamenumT gNum, std::size_t maxPly) const;
187 std::vector<scid::core::FullMove> mainlineMoves(
188 const IndexEntry* ie, std::size_t maxPly) const;
189 std::string moveSAN(gamenumT gNum, int plyToSkip, int count) const;
190 std::string moveSAN(const IndexEntry* ie, int plyToSkip, int count) const;
191 std::pair<scid::core::errorT, size_t>
192 replaceGameDates(HFilter filter, const Progress& progress,
193 scid::core::dateT oldDate, scid::core::dateT newDate);
194 std::pair<scid::core::errorT, size_t>
195 replaceGameEventDates(HFilter filter, const Progress& progress,
196 scid::core::dateT oldDate,
197 scid::core::dateT newDate);
198 std::pair<scid::core::errorT, size_t>
199 setPlayerRatings(HFilter filter, const Progress& progress, idNumberT player,
200 scid::core::ratingT rating,
201 scid::core::ratingTypeT ratingType);
202 template <typename TRatingResolver>
203 std::pair<scid::core::errorT, RatingUpdateStats> updatePlayerRatings(
204 HFilter filter, const Progress& progress, bool overwrite,
205 bool saveRatings, TRatingResolver ratingFor);
206 scid::core::errorT searchBoard(const IndexEntry& ie,
207 scid::core::Game& game,
208 scid::core::Position* pos,
209 scid::core::Position* posFlip,
210 bool useVariations,
211 bool possibleMatch,
212 bool possibleFlippedMatch,
213 gameExactMatchT searchType,
214 scid::core::uint& ply) const;
215 scid::core::errorT searchBoard(gamenumT gNum,
216 scid::core::Game& game,
217 scid::core::Position* pos,
218 scid::core::Position* posFlip,
219 bool useVariations,
220 bool possibleMatch,
221 bool possibleFlippedMatch,
222 gameExactMatchT searchType,
223 scid::core::uint& ply) const;
224 bool materialSearchMatch(const IndexEntry& ie, bool possibleMatch,
225 bool possibleFlippedMatch,
226 scid::core::byte* min, scid::core::byte* max,
227 scid::core::byte* minFlipped,
228 scid::core::byte* maxFlipped,
229 patternT* patterns, std::size_t patternCount,
230 patternT* flippedPatterns,
231 std::size_t flippedPatternCount, int minPly,
232 int maxPly, int matchLength, bool oppBishops,
233 bool sameBishops, int minDiff,
234 int maxDiff) const;
235 bool materialSearchMatch(gamenumT gNum, bool possibleMatch,
236 bool possibleFlippedMatch,
237 scid::core::byte* min, scid::core::byte* max,
238 scid::core::byte* minFlipped,
239 scid::core::byte* maxFlipped,
240 patternT* patterns, std::size_t patternCount,
241 patternT* flippedPatterns,
242 std::size_t flippedPatternCount, int minPly,
243 int maxPly, int matchLength, bool oppBishops,
244 bool sameBishops, int minDiff,
245 int maxDiff) const;
246 bool setPositionSearchFilter(const scid::core::Position& pos,
247 HFilter& filter,
248 const Progress& progress) const;
249
250 scid::core::errorT importGames(const scidBaseT* srcBase, const HFilter& filter,
251 const Progress& progress);
252 scid::core::errorT importGames(std::string_view dbType, const char* filename,
253 const Progress& progress, std::string& errorMsg);
254
263 scid::core::errorT saveGame(scid::core::Game const& game, const char* scidFlags,
264 gamenumT replacedGameId = INVALID_GAMEID);
265 scid::core::errorT addGame(scid::core::Game const& game, const char* scidFlags) {
266 return saveGame(game, scidFlags, INVALID_GAMEID);
267 }
268
269 bool getFlag(scid::core::uint flag, scid::core::uint gNum) const {
270 return idx->GetEntry(gNum)->GetFlag(flag);
271 }
272 scid::core::errorT setFlag(bool value, scid::core::uint flag, scid::core::uint gNum);
273 scid::core::errorT setFlags(bool value, scid::core::uint flag, const HFilter& filter);
274 scid::core::errorT invertFlag(scid::core::uint flag, scid::core::uint gNum);
275 scid::core::errorT invertFlags(scid::core::uint flag, const HFilter& filter);
276
282 std::string newFilter();
283 void deleteFilter(const char* filterId);
284 HFilter getFilter(std::string_view filterId) const;
285 HFilter defaultFilter() const { return HFilter(dbFilter); }
286 gamenumT defaultFilterCount() const { return dbFilter->Count(); }
287 scid::core::byte defaultFilterGet(gamenumT g) const { return dbFilter->Get(g); }
288 void defaultFilterSet(gamenumT g, scid::core::byte value) { dbFilter->Set(g, value); }
289 void defaultFilterFill(scid::core::byte value) { dbFilter->Fill(value); }
290 uint64_t cacheInvalidationToken() const { return cacheInvalidationToken_; }
291
299 std::string composeFilter(std::string_view mainFilter,
300 std::string_view maskFilter) const;
301
305 std::pair<std::string, std::string>
306 getFilterComponents(std::string_view filterId) const;
307
308 const Stats& getStats() const;
309 std::vector<TreeNode> getTreeStat(const HFilter& filter) const;
310 scid::core::uint getNameFreq(nameT nt, idNumberT id) {
311 if (nameFreq_[nt].size() == 0)
312 nameFreq_ = getNameBase()->calcNameFreq(*idx);
313 return nameFreq_[nt][id];
314 }
315
316 scid::core::errorT getCompactStat(unsigned long long* n_deleted,
317 unsigned long long* n_unused,
318 unsigned long long* n_sparse,
319 unsigned long long* n_badNameId);
320 scid::core::errorT compact(const Progress& progress);
321
329 bool createSortCache(const char* criteria);
330
339 void releaseSortCache(const char* criteria);
340
358 size_t listGames(const char* criteria, size_t start, size_t count,
359 const HFilter& filter, gamenumT* destCont);
360
373 size_t sortedPosition(const char* criteria, const HFilter& filter,
374 gamenumT gameId);
375
392 template <typename TInitFunc, typename TMapFunc>
393 std::pair<scid::core::errorT, size_t>
394 transformNames(nameT nt, HFilter hfilter, const Progress& progress,
395 const std::vector<std::string>& newNames, TInitFunc fnInit,
396 TMapFunc getID);
397
406 std::pair<scid::core::errorT, size_t>
407 stripGames(HFilter hfilter, const Progress& progress,
408 std::vector<std::string_view> const& removeTags);
409
410 std::unique_ptr<gamenumT[]> extractDuplicates() {
411 return std::move(duplicates_);
412 }
413 void setDuplicates(std::unique_ptr<gamenumT[]> duplicates) {
414 duplicates_ = std::move(duplicates);
415 }
416 gamenumT getDuplicates(gamenumT gNum) const {
417 return duplicates_ ? duplicates_[gNum] : 0;
418 }
419
420private:
421 struct Storage;
422
423 bool inUse; // true if the database is open (in use).
424 Filter* dbFilter;
425 std::unique_ptr<Storage> storage_;
426 Index* idx;
427 NameBase* nb_;
428 fileModeT fileMode_; // Read-only, write-only, or both.
429 std::vector<std::pair<std::string, Filter*>> filters_;
430 mutable Filter all_filter_{0};
431 mutable Stats* stats_;
432 std::array<std::vector<int>, NUM_NAME_TYPES> nameFreq_;
433 // For each game: idx of duplicate game + 1 (0 if there is no duplicate).
434 std::unique_ptr<gamenumT[]> duplicates_;
435 std::vector<std::pair<std::string, SortCache*>> sortCaches_;
436 mutable std::unordered_map<idNumberT, scid::core::ratingT> peakEloCache_;
437 scid::core::errorT err_open_ = scid::core::OK;
438 uint64_t cacheInvalidationToken_ = 0;
439
440private:
441 friend class SearchPos;
442
443 static GameInfo makeGameInfo_(const IndexEntry& ie);
444 ByteBuffer gameData(const IndexEntry& ie) const;
445 GameView gameView(const IndexEntry* ie) const;
446 scid::core::errorT openHelper(std::string_view dbType, fileModeT mode,
447 const char* filename, const Progress& progress = {});
448
449 void clear();
450
454 scid::core::errorT beginTransaction();
455
461 scid::core::errorT endTransaction(gamenumT gameId = INVALID_GAMEID);
462
463 scid::core::errorT importGameHelper(const scidBaseT* sourceBase, scid::core::uint gNum);
464 scid::core::errorT saveGameData(IndexEntry const& ie, TagRoster const& tags,
465 ByteBuffer const& data, gamenumT replaced);
466 scid::core::errorT saveIndexEntry(IndexEntry const& ie, gamenumT replaced);
467 std::pair<scid::core::errorT, idNumberT> addName(nameT nt, const char* name);
468
469 SortCache* getSortCache(const char* criteria);
470
471 template <typename TOper>
472 std::pair<scid::core::errorT, size_t>
473 transformIndex(HFilter hfilter, const Progress& progress, TOper entry_op) {
474 if (auto errModify = beginTransaction())
475 return {errModify, 0};
476
477 auto res = transformIndex_(hfilter, progress, entry_op);
478 auto err = endTransaction();
479 res.first = (res.first == scid::core::OK) ? err : res.first;
480 return res;
481 }
482
493 template <typename TOper>
494 std::pair<scid::core::errorT, size_t>
495 transformIndex_(HFilter hfilter, const Progress& progress, TOper entry_op) {
496 size_t nCorrections = 0;
497 size_t iProg = 0;
498 size_t totProg = hfilter->size();
499 for (auto& gnum : hfilter) {
500 if ((++iProg % 8192 == 0) && !progress.report(iProg, totProg))
501 return std::make_pair(scid::core::ERROR_UserCancel, nCorrections);
502
503 IndexEntry newIE = *getIndexEntry(gnum);
504 if (!entry_op(newIE))
505 continue;
506
507 auto err = saveIndexEntry(newIE, gnum);
508 if (err != scid::core::OK)
509 return std::make_pair(err, nCorrections);
510
511 ++nCorrections;
512 }
513 return std::make_pair(scid::core::OK, nCorrections);
514 }
515};
516
517template <typename TInitFunc, typename TMapFunc>
518std::pair<scid::core::errorT, size_t>
519scidBaseT::transformNames(nameT nt, HFilter hfilter, const Progress& progress,
520 const std::vector<std::string>& newNames,
521 TInitFunc initFunc, TMapFunc getNewID) {
522 if (auto errModify = beginTransaction())
523 return {errModify, 0};
524
525 std::vector<idNumberT> nameIDs(newNames.size());
526 auto it = nameIDs.begin();
527 for (auto& name : newNames) {
528 auto id = addName(nt, name.c_str());
529 if (id.first != scid::core::OK) {
530 endTransaction();
531 return std::make_pair(id.first, size_t(0));
532 }
533 *it++ = id.second;
534 }
535
536 initFunc(nameIDs);
537
538 auto res = transformIndex_(hfilter, progress, [&](IndexEntry& ie) {
539 const IndexEntry& ie_const = ie;
540 idNumberT oldID;
541 idNumberT oldBlackID = 0;
542 idNumberT newBlackID = 0;
543 switch (nt) {
544 case NAME_PLAYER:
545 oldID = ie_const.GetWhite();
546 oldBlackID = ie_const.GetBlack();
547 newBlackID = getNewID(oldBlackID, makeGameInfo_(ie_const));
548 break;
549 case NAME_EVENT:
550 oldID = ie_const.GetEvent();
551 break;
552 case NAME_SITE:
553 oldID = ie_const.GetSite();
554 break;
555 default:
556 ASSERT(nt == NAME_ROUND);
557 oldID = ie_const.GetRound();
558 }
559 const auto newID = getNewID(oldID, makeGameInfo_(ie_const));
560 if (oldID == newID && oldBlackID == newBlackID)
561 return false;
562
563 switch (nt) {
564 case NAME_PLAYER:
565 ie.SetWhite(newID);
566 ie.SetBlack(newBlackID);
567 break;
568 case NAME_EVENT:
569 ie.SetEvent(newID);
570 break;
571 case NAME_SITE:
572 ie.SetSite(newID);
573 break;
574 default:
575 ASSERT(nt == NAME_ROUND);
576 ie.SetRound(newID);
577 }
578 return true;
579 });
580
581 auto err = endTransaction();
582 res.first = (res.first == scid::core::OK) ? err : res.first;
583 return res;
584}
585
586template <typename TRatingResolver>
587std::pair<scid::core::errorT, scidBaseT::RatingUpdateStats>
588scidBaseT::updatePlayerRatings(HFilter filter, const Progress& progress,
589 bool overwrite, bool saveRatings,
590 TRatingResolver ratingFor) {
591 RatingUpdateStats stats;
592 auto entry_op = [&](IndexEntry& ie) {
593 const auto date = ie.GetDate();
594 const auto whiteElo = (!overwrite && ie.GetWhiteElo() != 0)
595 ? 0
596 : ratingFor(ie.GetWhite(), date);
597 const auto blackElo = (!overwrite && ie.GetBlackElo() != 0)
598 ? 0
599 : ratingFor(ie.GetBlack(), date);
600 const auto changes = (whiteElo != 0 ? 1 : 0) + (blackElo != 0 ? 1 : 0);
601 if (changes == 0)
602 return false;
603
604 stats.changedRatings += changes;
605 stats.changedGames++;
606 if (!saveRatings)
607 return false;
608
609 if (whiteElo != 0) {
610 ie.SetWhiteElo(whiteElo);
611 ie.SetWhiteRatingType(scid::core::RATING_Elo);
612 }
613 if (blackElo != 0) {
614 ie.SetBlackElo(blackElo);
615 ie.SetBlackRatingType(scid::core::RATING_Elo);
616 }
617 return true;
618 };
619
620 if (!saveRatings) {
621 auto res = transformIndex_(filter, progress, entry_op);
622 return {res.first, stats};
623 }
624
625 auto res = transformIndex(filter, progress, entry_op);
626 return {res.first, stats};
627}
628
629} // namespace scid::database
630#endif
board.h
Chess board constants, piece helpers, square geometry, and directions.
scid::core::Game
Definition game.h:85
scid::core::Position
Definition position.h:44
scid::database::Filter::Fill
void Fill(scid::core::byte value)
Sets all values.
Definition hfilter.h:112
scid::database::Filter::Get
scid::core::byte Get(gamenumT index) const
Gets the value at index.
Definition hfilter.h:86
scid::database::Filter::Count
gamenumT Count() const
Return the number of nonzero values in filter.
Definition hfilter.h:57
scid::database::Filter::Set
void Set(gamenumT index, scid::core::byte value)
Sets the value at index.
Definition hfilter.h:92
scid::database::HFilter
Definition hfilter.h:152
scid::database::IndexEntry
Definition indexentry.h:51
scid::database::Index::GetNumGames
gamenumT GetNumGames() const
Header getter functions.
scid::database::NameBase::calcNameFreq
std::array< std::vector< int >, NUM_NAME_TYPES > calcNameFreq(Index const &idx) const
Counts how many times each name is used.
Definition namebase.h:231
scid::database::Progress
Definition misc.h:66
fullmove.h
Compact encoded move representation.
scid::database
class StrRange - parse a string interpreting its content as 1 or 2 integers separated by whitespace.
Definition common.h:30
scid::database::patternT
Definition scidbase.h:55
scid::database::scidBaseT::Stats
Definition scidbase.h:75
scid::database::scidBaseT
Definition scidbase.h:69
scid::database::scidBaseT::newFilter
std::string newFilter()
A Filter is a selection of games, usually obtained searching the database.
scid::database::scidBaseT::composeFilter
std::string composeFilter(std::string_view mainFilter, std::string_view maskFilter) const
A composed filter is a special construct created combining two filters and includes only the games co...
scid::database::scidBaseT::stripGames
std::pair< scid::core::errorT, size_t > stripGames(HFilter hfilter, const Progress &progress, std::vector< std::string_view > const &removeTags)
Strip the games included in hfilter.
scid::database::scidBaseT::createSortCache
bool createSortCache(const char *criteria)
Increment the reference count of a SortCache object matching criteria.
scid::database::scidBaseT::sortedPosition
size_t sortedPosition(const char *criteria, const HFilter &filter, gamenumT gameId)
Get the sorted position of a game.
scid::database::scidBaseT::peakElo
scid::core::ratingT peakElo(idNumberT playerID) const
Return the highest elo of the player (in the database's games)
Definition scidbase.h:152
scid::database::scidBaseT::transformNames
std::pair< scid::core::errorT, size_t > transformNames(nameT nt, HFilter hfilter, const Progress &progress, const std::vector< std::string > &newNames, TInitFunc fnInit, TMapFunc getID)
Transform the names of the games included in hfilter.
Definition scidbase.h:519
scid::database::scidBaseT::getExtraInfo
std::vector< std::pair< const char *, std::string > > getExtraInfo() const
Returns a vector of tag pairs containing extra information about the database (type,...
scid::database::scidBaseT::getFilterComponents
std::pair< std::string, std::string > getFilterComponents(std::string_view filterId) const
Get the components of a composed filter.
scid::database::scidBaseT::listGames
size_t listGames(const char *criteria, size_t start, size_t count, const HFilter &filter, gamenumT *destCont)
Retrieve a list of ordered game indexes sorted by criteria.
scid::database::scidBaseT::saveGame
scid::core::errorT saveGame(scid::core::Game const &game, const char *scidFlags, gamenumT replacedGameId=INVALID_GAMEID)
Add or replace a game into the database.
scid::database::scidBaseT::releaseSortCache
void releaseSortCache(const char *criteria)
Decrement the reference count of the SortCache object matching criteria.
scid::database::scidBaseT::setExtraInfo
scid::core::errorT setExtraInfo(const char *tagname, const char *new_value)
Store an extra information about the database (type, description, etc..)