fault_injection_test.cc raw
1 // Copyright 2014 The LevelDB Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file. See the AUTHORS file for names of contributors.
4
5 // This test uses a custom Env to keep track of the state of a filesystem as of
6 // the last "sync". It then checks for data loss errors by purposely dropping
7 // file data (or entire files) not protected by a "sync".
8
9 #include <map>
10 #include <set>
11
12 #include "db/db_impl.h"
13 #include "db/filename.h"
14 #include "db/log_format.h"
15 #include "db/version_set.h"
16 #include "leveldb/cache.h"
17 #include "leveldb/db.h"
18 #include "leveldb/env.h"
19 #include "leveldb/table.h"
20 #include "leveldb/write_batch.h"
21 #include "port/port.h"
22 #include "port/thread_annotations.h"
23 #include "util/logging.h"
24 #include "util/mutexlock.h"
25 #include "util/testharness.h"
26 #include "util/testutil.h"
27
28 namespace leveldb {
29
30 static const int kValueSize = 1000;
31 static const int kMaxNumValues = 2000;
32 static const size_t kNumIterations = 3;
33
34 class FaultInjectionTestEnv;
35
36 namespace {
37
38 // Assume a filename, and not a directory name like "/foo/bar/"
39 static std::string GetDirName(const std::string& filename) {
40 size_t found = filename.find_last_of("/\\");
41 if (found == std::string::npos) {
42 return "";
43 } else {
44 return filename.substr(0, found);
45 }
46 }
47
48 Status SyncDir(const std::string& dir) {
49 // As this is a test it isn't required to *actually* sync this directory.
50 return Status::OK();
51 }
52
53 // A basic file truncation function suitable for this test.
54 Status Truncate(const std::string& filename, uint64_t length) {
55 leveldb::Env* env = leveldb::Env::Default();
56
57 SequentialFile* orig_file;
58 Status s = env->NewSequentialFile(filename, &orig_file);
59 if (!s.ok()) return s;
60
61 char* scratch = new char[length];
62 leveldb::Slice result;
63 s = orig_file->Read(length, &result, scratch);
64 delete orig_file;
65 if (s.ok()) {
66 std::string tmp_name = GetDirName(filename) + "/truncate.tmp";
67 WritableFile* tmp_file;
68 s = env->NewWritableFile(tmp_name, &tmp_file);
69 if (s.ok()) {
70 s = tmp_file->Append(result);
71 delete tmp_file;
72 if (s.ok()) {
73 s = env->RenameFile(tmp_name, filename);
74 } else {
75 env->DeleteFile(tmp_name);
76 }
77 }
78 }
79
80 delete[] scratch;
81
82 return s;
83 }
84
85 struct FileState {
86 std::string filename_;
87 int64_t pos_;
88 int64_t pos_at_last_sync_;
89 int64_t pos_at_last_flush_;
90
91 FileState(const std::string& filename)
92 : filename_(filename),
93 pos_(-1),
94 pos_at_last_sync_(-1),
95 pos_at_last_flush_(-1) {}
96
97 FileState() : pos_(-1), pos_at_last_sync_(-1), pos_at_last_flush_(-1) {}
98
99 bool IsFullySynced() const { return pos_ <= 0 || pos_ == pos_at_last_sync_; }
100
101 Status DropUnsyncedData() const;
102 };
103
104 } // anonymous namespace
105
106 // A wrapper around WritableFile which informs another Env whenever this file
107 // is written to or sync'ed.
108 class TestWritableFile : public WritableFile {
109 public:
110 TestWritableFile(const FileState& state, WritableFile* f,
111 FaultInjectionTestEnv* env);
112 ~TestWritableFile() override;
113 Status Append(const Slice& data) override;
114 Status Close() override;
115 Status Flush() override;
116 Status Sync() override;
117 std::string GetName() const override { return ""; }
118
119 private:
120 FileState state_;
121 WritableFile* target_;
122 bool writable_file_opened_;
123 FaultInjectionTestEnv* env_;
124
125 Status SyncParent();
126 };
127
128 class FaultInjectionTestEnv : public EnvWrapper {
129 public:
130 FaultInjectionTestEnv()
131 : EnvWrapper(Env::Default()), filesystem_active_(true) {}
132 ~FaultInjectionTestEnv() override = default;
133 Status NewWritableFile(const std::string& fname,
134 WritableFile** result) override;
135 Status NewAppendableFile(const std::string& fname,
136 WritableFile** result) override;
137 Status DeleteFile(const std::string& f) override;
138 Status RenameFile(const std::string& s, const std::string& t) override;
139
140 void WritableFileClosed(const FileState& state);
141 Status DropUnsyncedFileData();
142 Status DeleteFilesCreatedAfterLastDirSync();
143 void DirWasSynced();
144 bool IsFileCreatedSinceLastDirSync(const std::string& filename);
145 void ResetState();
146 void UntrackFile(const std::string& f);
147 // Setting the filesystem to inactive is the test equivalent to simulating a
148 // system reset. Setting to inactive will freeze our saved filesystem state so
149 // that it will stop being recorded. It can then be reset back to the state at
150 // the time of the reset.
151 bool IsFilesystemActive() LOCKS_EXCLUDED(mutex_) {
152 MutexLock l(&mutex_);
153 return filesystem_active_;
154 }
155 void SetFilesystemActive(bool active) LOCKS_EXCLUDED(mutex_) {
156 MutexLock l(&mutex_);
157 filesystem_active_ = active;
158 }
159
160 private:
161 port::Mutex mutex_;
162 std::map<std::string, FileState> db_file_state_ GUARDED_BY(mutex_);
163 std::set<std::string> new_files_since_last_dir_sync_ GUARDED_BY(mutex_);
164 bool filesystem_active_ GUARDED_BY(mutex_); // Record flushes, syncs, writes
165 };
166
167 TestWritableFile::TestWritableFile(const FileState& state, WritableFile* f,
168 FaultInjectionTestEnv* env)
169 : state_(state), target_(f), writable_file_opened_(true), env_(env) {
170 assert(f != nullptr);
171 }
172
173 TestWritableFile::~TestWritableFile() {
174 if (writable_file_opened_) {
175 Close();
176 }
177 delete target_;
178 }
179
180 Status TestWritableFile::Append(const Slice& data) {
181 Status s = target_->Append(data);
182 if (s.ok() && env_->IsFilesystemActive()) {
183 state_.pos_ += data.size();
184 }
185 return s;
186 }
187
188 Status TestWritableFile::Close() {
189 writable_file_opened_ = false;
190 Status s = target_->Close();
191 if (s.ok()) {
192 env_->WritableFileClosed(state_);
193 }
194 return s;
195 }
196
197 Status TestWritableFile::Flush() {
198 Status s = target_->Flush();
199 if (s.ok() && env_->IsFilesystemActive()) {
200 state_.pos_at_last_flush_ = state_.pos_;
201 }
202 return s;
203 }
204
205 Status TestWritableFile::SyncParent() {
206 Status s = SyncDir(GetDirName(state_.filename_));
207 if (s.ok()) {
208 env_->DirWasSynced();
209 }
210 return s;
211 }
212
213 Status TestWritableFile::Sync() {
214 if (!env_->IsFilesystemActive()) {
215 return Status::OK();
216 }
217 // Ensure new files referred to by the manifest are in the filesystem.
218 Status s = target_->Sync();
219 if (s.ok()) {
220 state_.pos_at_last_sync_ = state_.pos_;
221 }
222 if (env_->IsFileCreatedSinceLastDirSync(state_.filename_)) {
223 Status ps = SyncParent();
224 if (s.ok() && !ps.ok()) {
225 s = ps;
226 }
227 }
228 return s;
229 }
230
231 Status FaultInjectionTestEnv::NewWritableFile(const std::string& fname,
232 WritableFile** result) {
233 WritableFile* actual_writable_file;
234 Status s = target()->NewWritableFile(fname, &actual_writable_file);
235 if (s.ok()) {
236 FileState state(fname);
237 state.pos_ = 0;
238 *result = new TestWritableFile(state, actual_writable_file, this);
239 // NewWritableFile doesn't append to files, so if the same file is
240 // opened again then it will be truncated - so forget our saved
241 // state.
242 UntrackFile(fname);
243 MutexLock l(&mutex_);
244 new_files_since_last_dir_sync_.insert(fname);
245 }
246 return s;
247 }
248
249 Status FaultInjectionTestEnv::NewAppendableFile(const std::string& fname,
250 WritableFile** result) {
251 WritableFile* actual_writable_file;
252 Status s = target()->NewAppendableFile(fname, &actual_writable_file);
253 if (s.ok()) {
254 FileState state(fname);
255 state.pos_ = 0;
256 {
257 MutexLock l(&mutex_);
258 if (db_file_state_.count(fname) == 0) {
259 new_files_since_last_dir_sync_.insert(fname);
260 } else {
261 state = db_file_state_[fname];
262 }
263 }
264 *result = new TestWritableFile(state, actual_writable_file, this);
265 }
266 return s;
267 }
268
269 Status FaultInjectionTestEnv::DropUnsyncedFileData() {
270 Status s;
271 MutexLock l(&mutex_);
272 for (const auto& kvp : db_file_state_) {
273 if (!s.ok()) {
274 break;
275 }
276 const FileState& state = kvp.second;
277 if (!state.IsFullySynced()) {
278 s = state.DropUnsyncedData();
279 }
280 }
281 return s;
282 }
283
284 void FaultInjectionTestEnv::DirWasSynced() {
285 MutexLock l(&mutex_);
286 new_files_since_last_dir_sync_.clear();
287 }
288
289 bool FaultInjectionTestEnv::IsFileCreatedSinceLastDirSync(
290 const std::string& filename) {
291 MutexLock l(&mutex_);
292 return new_files_since_last_dir_sync_.find(filename) !=
293 new_files_since_last_dir_sync_.end();
294 }
295
296 void FaultInjectionTestEnv::UntrackFile(const std::string& f) {
297 MutexLock l(&mutex_);
298 db_file_state_.erase(f);
299 new_files_since_last_dir_sync_.erase(f);
300 }
301
302 Status FaultInjectionTestEnv::DeleteFile(const std::string& f) {
303 Status s = EnvWrapper::DeleteFile(f);
304 ASSERT_OK(s);
305 if (s.ok()) {
306 UntrackFile(f);
307 }
308 return s;
309 }
310
311 Status FaultInjectionTestEnv::RenameFile(const std::string& s,
312 const std::string& t) {
313 Status ret = EnvWrapper::RenameFile(s, t);
314
315 if (ret.ok()) {
316 MutexLock l(&mutex_);
317 if (db_file_state_.find(s) != db_file_state_.end()) {
318 db_file_state_[t] = db_file_state_[s];
319 db_file_state_.erase(s);
320 }
321
322 if (new_files_since_last_dir_sync_.erase(s) != 0) {
323 assert(new_files_since_last_dir_sync_.find(t) ==
324 new_files_since_last_dir_sync_.end());
325 new_files_since_last_dir_sync_.insert(t);
326 }
327 }
328
329 return ret;
330 }
331
332 void FaultInjectionTestEnv::ResetState() {
333 // Since we are not destroying the database, the existing files
334 // should keep their recorded synced/flushed state. Therefore
335 // we do not reset db_file_state_ and new_files_since_last_dir_sync_.
336 SetFilesystemActive(true);
337 }
338
339 Status FaultInjectionTestEnv::DeleteFilesCreatedAfterLastDirSync() {
340 // Because DeleteFile access this container make a copy to avoid deadlock
341 mutex_.Lock();
342 std::set<std::string> new_files(new_files_since_last_dir_sync_.begin(),
343 new_files_since_last_dir_sync_.end());
344 mutex_.Unlock();
345 Status status;
346 for (const auto& new_file : new_files) {
347 Status delete_status = DeleteFile(new_file);
348 if (!delete_status.ok() && status.ok()) {
349 status = std::move(delete_status);
350 }
351 }
352 return status;
353 }
354
355 void FaultInjectionTestEnv::WritableFileClosed(const FileState& state) {
356 MutexLock l(&mutex_);
357 db_file_state_[state.filename_] = state;
358 }
359
360 Status FileState::DropUnsyncedData() const {
361 int64_t sync_pos = pos_at_last_sync_ == -1 ? 0 : pos_at_last_sync_;
362 return Truncate(filename_, sync_pos);
363 }
364
365 class FaultInjectionTest {
366 public:
367 enum ExpectedVerifResult { VAL_EXPECT_NO_ERROR, VAL_EXPECT_ERROR };
368 enum ResetMethod { RESET_DROP_UNSYNCED_DATA, RESET_DELETE_UNSYNCED_FILES };
369
370 FaultInjectionTestEnv* env_;
371 std::string dbname_;
372 Cache* tiny_cache_;
373 Options options_;
374 DB* db_;
375
376 FaultInjectionTest()
377 : env_(new FaultInjectionTestEnv),
378 tiny_cache_(NewLRUCache(100)),
379 db_(nullptr) {
380 dbname_ = test::TmpDir() + "/fault_test";
381 DestroyDB(dbname_, Options()); // Destroy any db from earlier run
382 options_.reuse_logs = true;
383 options_.env = env_;
384 options_.paranoid_checks = true;
385 options_.block_cache = tiny_cache_;
386 options_.create_if_missing = true;
387 }
388
389 ~FaultInjectionTest() {
390 CloseDB();
391 DestroyDB(dbname_, Options());
392 delete tiny_cache_;
393 delete env_;
394 }
395
396 void ReuseLogs(bool reuse) { options_.reuse_logs = reuse; }
397
398 void Build(int start_idx, int num_vals) {
399 std::string key_space, value_space;
400 WriteBatch batch;
401 for (int i = start_idx; i < start_idx + num_vals; i++) {
402 Slice key = Key(i, &key_space);
403 batch.Clear();
404 batch.Put(key, Value(i, &value_space));
405 WriteOptions options;
406 ASSERT_OK(db_->Write(options, &batch));
407 }
408 }
409
410 Status ReadValue(int i, std::string* val) const {
411 std::string key_space, value_space;
412 Slice key = Key(i, &key_space);
413 Value(i, &value_space);
414 ReadOptions options;
415 return db_->Get(options, key, val);
416 }
417
418 Status Verify(int start_idx, int num_vals,
419 ExpectedVerifResult expected) const {
420 std::string val;
421 std::string value_space;
422 Status s;
423 for (int i = start_idx; i < start_idx + num_vals && s.ok(); i++) {
424 Value(i, &value_space);
425 s = ReadValue(i, &val);
426 if (expected == VAL_EXPECT_NO_ERROR) {
427 if (s.ok()) {
428 ASSERT_EQ(value_space, val);
429 }
430 } else if (s.ok()) {
431 fprintf(stderr, "Expected an error at %d, but was OK\n", i);
432 s = Status::IOError(dbname_, "Expected value error:");
433 } else {
434 s = Status::OK(); // An expected error
435 }
436 }
437 return s;
438 }
439
440 // Return the ith key
441 Slice Key(int i, std::string* storage) const {
442 char buf[100];
443 snprintf(buf, sizeof(buf), "%016d", i);
444 storage->assign(buf, strlen(buf));
445 return Slice(*storage);
446 }
447
448 // Return the value to associate with the specified key
449 Slice Value(int k, std::string* storage) const {
450 Random r(k);
451 return test::RandomString(&r, kValueSize, storage);
452 }
453
454 Status OpenDB() {
455 delete db_;
456 db_ = nullptr;
457 env_->ResetState();
458 return DB::Open(options_, dbname_, &db_);
459 }
460
461 void CloseDB() {
462 delete db_;
463 db_ = nullptr;
464 }
465
466 void DeleteAllData() {
467 Iterator* iter = db_->NewIterator(ReadOptions());
468 for (iter->SeekToFirst(); iter->Valid(); iter->Next()) {
469 ASSERT_OK(db_->Delete(WriteOptions(), iter->key()));
470 }
471
472 delete iter;
473 }
474
475 void ResetDBState(ResetMethod reset_method) {
476 switch (reset_method) {
477 case RESET_DROP_UNSYNCED_DATA:
478 ASSERT_OK(env_->DropUnsyncedFileData());
479 break;
480 case RESET_DELETE_UNSYNCED_FILES:
481 ASSERT_OK(env_->DeleteFilesCreatedAfterLastDirSync());
482 break;
483 default:
484 assert(false);
485 }
486 }
487
488 void PartialCompactTestPreFault(int num_pre_sync, int num_post_sync) {
489 DeleteAllData();
490 Build(0, num_pre_sync);
491 db_->CompactRange(nullptr, nullptr);
492 Build(num_pre_sync, num_post_sync);
493 }
494
495 void PartialCompactTestReopenWithFault(ResetMethod reset_method,
496 int num_pre_sync, int num_post_sync) {
497 env_->SetFilesystemActive(false);
498 CloseDB();
499 ResetDBState(reset_method);
500 ASSERT_OK(OpenDB());
501 ASSERT_OK(Verify(0, num_pre_sync, FaultInjectionTest::VAL_EXPECT_NO_ERROR));
502 ASSERT_OK(Verify(num_pre_sync, num_post_sync,
503 FaultInjectionTest::VAL_EXPECT_ERROR));
504 }
505
506 void NoWriteTestPreFault() {}
507
508 void NoWriteTestReopenWithFault(ResetMethod reset_method) {
509 CloseDB();
510 ResetDBState(reset_method);
511 ASSERT_OK(OpenDB());
512 }
513
514 void DoTest() {
515 Random rnd(0);
516 ASSERT_OK(OpenDB());
517 for (size_t idx = 0; idx < kNumIterations; idx++) {
518 int num_pre_sync = rnd.Uniform(kMaxNumValues);
519 int num_post_sync = rnd.Uniform(kMaxNumValues);
520
521 PartialCompactTestPreFault(num_pre_sync, num_post_sync);
522 PartialCompactTestReopenWithFault(RESET_DROP_UNSYNCED_DATA, num_pre_sync,
523 num_post_sync);
524
525 NoWriteTestPreFault();
526 NoWriteTestReopenWithFault(RESET_DROP_UNSYNCED_DATA);
527
528 PartialCompactTestPreFault(num_pre_sync, num_post_sync);
529 // No new files created so we expect all values since no files will be
530 // dropped.
531 PartialCompactTestReopenWithFault(RESET_DELETE_UNSYNCED_FILES,
532 num_pre_sync + num_post_sync, 0);
533
534 NoWriteTestPreFault();
535 NoWriteTestReopenWithFault(RESET_DELETE_UNSYNCED_FILES);
536 }
537 }
538 };
539
540 TEST(FaultInjectionTest, FaultTestNoLogReuse) {
541 ReuseLogs(false);
542 DoTest();
543 }
544
545 TEST(FaultInjectionTest, FaultTestWithLogReuse) {
546 ReuseLogs(true);
547 DoTest();
548 }
549
550 } // namespace leveldb
551
552 int main(int argc, char** argv) { return leveldb::test::RunAllTests(); }
553