corruption_test.cc raw
1 // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file. See the AUTHORS file for names of contributors.
4
5 #include <sys/types.h>
6
7 #include "db/db_impl.h"
8 #include "db/filename.h"
9 #include "db/log_format.h"
10 #include "db/version_set.h"
11 #include "leveldb/cache.h"
12 #include "leveldb/db.h"
13 #include "leveldb/table.h"
14 #include "leveldb/write_batch.h"
15 #include "util/logging.h"
16 #include "util/testharness.h"
17 #include "util/testutil.h"
18
19 namespace leveldb {
20
21 static const int kValueSize = 1000;
22
23 class CorruptionTest {
24 public:
25 CorruptionTest()
26 : db_(nullptr),
27 dbname_("/memenv/corruption_test"),
28 tiny_cache_(NewLRUCache(100)) {
29 options_.env = &env_;
30 options_.block_cache = tiny_cache_;
31 DestroyDB(dbname_, options_);
32
33 options_.create_if_missing = true;
34 Reopen();
35 options_.create_if_missing = false;
36 }
37
38 ~CorruptionTest() {
39 delete db_;
40 delete tiny_cache_;
41 }
42
43 Status TryReopen() {
44 delete db_;
45 db_ = nullptr;
46 return DB::Open(options_, dbname_, &db_);
47 }
48
49 void Reopen() { ASSERT_OK(TryReopen()); }
50
51 void RepairDB() {
52 delete db_;
53 db_ = nullptr;
54 ASSERT_OK(::leveldb::RepairDB(dbname_, options_));
55 }
56
57 void Build(int n) {
58 std::string key_space, value_space;
59 WriteBatch batch;
60 for (int i = 0; i < n; i++) {
61 // if ((i % 100) == 0) fprintf(stderr, "@ %d of %d\n", i, n);
62 Slice key = Key(i, &key_space);
63 batch.Clear();
64 batch.Put(key, Value(i, &value_space));
65 WriteOptions options;
66 // Corrupt() doesn't work without this sync on windows; stat reports 0 for
67 // the file size.
68 if (i == n - 1) {
69 options.sync = true;
70 }
71 ASSERT_OK(db_->Write(options, &batch));
72 }
73 }
74
75 void Check(int min_expected, int max_expected) {
76 int next_expected = 0;
77 int missed = 0;
78 int bad_keys = 0;
79 int bad_values = 0;
80 int correct = 0;
81 std::string value_space;
82 Iterator* iter = db_->NewIterator(ReadOptions());
83 for (iter->SeekToFirst(); iter->Valid(); iter->Next()) {
84 uint64_t key;
85 Slice in(iter->key());
86 if (in == "" || in == "~") {
87 // Ignore boundary keys.
88 continue;
89 }
90 if (!ConsumeDecimalNumber(&in, &key) || !in.empty() ||
91 key < next_expected) {
92 bad_keys++;
93 continue;
94 }
95 missed += (key - next_expected);
96 next_expected = key + 1;
97 if (iter->value() != Value(key, &value_space)) {
98 bad_values++;
99 } else {
100 correct++;
101 }
102 }
103 delete iter;
104
105 fprintf(stderr,
106 "expected=%d..%d; got=%d; bad_keys=%d; bad_values=%d; missed=%d\n",
107 min_expected, max_expected, correct, bad_keys, bad_values, missed);
108 ASSERT_LE(min_expected, correct);
109 ASSERT_GE(max_expected, correct);
110 }
111
112 void Corrupt(FileType filetype, int offset, int bytes_to_corrupt) {
113 // Pick file to corrupt
114 std::vector<std::string> filenames;
115 ASSERT_OK(env_.target()->GetChildren(dbname_, &filenames));
116 uint64_t number;
117 FileType type;
118 std::string fname;
119 int picked_number = -1;
120 for (size_t i = 0; i < filenames.size(); i++) {
121 if (ParseFileName(filenames[i], &number, &type) && type == filetype &&
122 int(number) > picked_number) { // Pick latest file
123 fname = dbname_ + "/" + filenames[i];
124 picked_number = number;
125 }
126 }
127 ASSERT_TRUE(!fname.empty()) << filetype;
128
129 uint64_t file_size;
130 ASSERT_OK(env_.target()->GetFileSize(fname, &file_size));
131
132 if (offset < 0) {
133 // Relative to end of file; make it absolute
134 if (-offset > file_size) {
135 offset = 0;
136 } else {
137 offset = file_size + offset;
138 }
139 }
140 if (offset > file_size) {
141 offset = file_size;
142 }
143 if (offset + bytes_to_corrupt > file_size) {
144 bytes_to_corrupt = file_size - offset;
145 }
146
147 // Do it
148 std::string contents;
149 Status s = ReadFileToString(env_.target(), fname, &contents);
150 ASSERT_TRUE(s.ok()) << s.ToString();
151 for (int i = 0; i < bytes_to_corrupt; i++) {
152 contents[i + offset] ^= 0x80;
153 }
154 s = WriteStringToFile(env_.target(), contents, fname);
155 ASSERT_TRUE(s.ok()) << s.ToString();
156 }
157
158 int Property(const std::string& name) {
159 std::string property;
160 int result;
161 if (db_->GetProperty(name, &property) &&
162 sscanf(property.c_str(), "%d", &result) == 1) {
163 return result;
164 } else {
165 return -1;
166 }
167 }
168
169 // Return the ith key
170 Slice Key(int i, std::string* storage) {
171 char buf[100];
172 snprintf(buf, sizeof(buf), "%016d", i);
173 storage->assign(buf, strlen(buf));
174 return Slice(*storage);
175 }
176
177 // Return the value to associate with the specified key
178 Slice Value(int k, std::string* storage) {
179 Random r(k);
180 return test::RandomString(&r, kValueSize, storage);
181 }
182
183 test::ErrorEnv env_;
184 Options options_;
185 DB* db_;
186
187 private:
188 std::string dbname_;
189 Cache* tiny_cache_;
190 };
191
192 TEST(CorruptionTest, Recovery) {
193 Build(100);
194 Check(100, 100);
195 Corrupt(kLogFile, 19, 1); // WriteBatch tag for first record
196 Corrupt(kLogFile, log::kBlockSize + 1000, 1); // Somewhere in second block
197 Reopen();
198
199 // The 64 records in the first two log blocks are completely lost.
200 Check(36, 36);
201 }
202
203 TEST(CorruptionTest, RecoverWriteError) {
204 env_.writable_file_error_ = true;
205 Status s = TryReopen();
206 ASSERT_TRUE(!s.ok());
207 }
208
209 TEST(CorruptionTest, NewFileErrorDuringWrite) {
210 // Do enough writing to force minor compaction
211 env_.writable_file_error_ = true;
212 const int num = 3 + (Options().write_buffer_size / kValueSize);
213 std::string value_storage;
214 Status s;
215 for (int i = 0; s.ok() && i < num; i++) {
216 WriteBatch batch;
217 batch.Put("a", Value(100, &value_storage));
218 s = db_->Write(WriteOptions(), &batch);
219 }
220 ASSERT_TRUE(!s.ok());
221 ASSERT_GE(env_.num_writable_file_errors_, 1);
222 env_.writable_file_error_ = false;
223 Reopen();
224 }
225
226 TEST(CorruptionTest, TableFile) {
227 Build(100);
228 DBImpl* dbi = reinterpret_cast<DBImpl*>(db_);
229 dbi->TEST_CompactMemTable();
230 dbi->TEST_CompactRange(0, nullptr, nullptr);
231 dbi->TEST_CompactRange(1, nullptr, nullptr);
232
233 Corrupt(kTableFile, 100, 1);
234 Check(90, 99);
235 }
236
237 TEST(CorruptionTest, TableFileRepair) {
238 options_.block_size = 2 * kValueSize; // Limit scope of corruption
239 options_.paranoid_checks = true;
240 Reopen();
241 Build(100);
242 DBImpl* dbi = reinterpret_cast<DBImpl*>(db_);
243 dbi->TEST_CompactMemTable();
244 dbi->TEST_CompactRange(0, nullptr, nullptr);
245 dbi->TEST_CompactRange(1, nullptr, nullptr);
246
247 Corrupt(kTableFile, 100, 1);
248 RepairDB();
249 Reopen();
250 Check(95, 99);
251 }
252
253 TEST(CorruptionTest, TableFileIndexData) {
254 Build(10000); // Enough to build multiple Tables
255 DBImpl* dbi = reinterpret_cast<DBImpl*>(db_);
256 dbi->TEST_CompactMemTable();
257
258 Corrupt(kTableFile, -2000, 500);
259 Reopen();
260 Check(5000, 9999);
261 }
262
263 TEST(CorruptionTest, MissingDescriptor) {
264 Build(1000);
265 RepairDB();
266 Reopen();
267 Check(1000, 1000);
268 }
269
270 TEST(CorruptionTest, SequenceNumberRecovery) {
271 ASSERT_OK(db_->Put(WriteOptions(), "foo", "v1"));
272 ASSERT_OK(db_->Put(WriteOptions(), "foo", "v2"));
273 ASSERT_OK(db_->Put(WriteOptions(), "foo", "v3"));
274 ASSERT_OK(db_->Put(WriteOptions(), "foo", "v4"));
275 ASSERT_OK(db_->Put(WriteOptions(), "foo", "v5"));
276 RepairDB();
277 Reopen();
278 std::string v;
279 ASSERT_OK(db_->Get(ReadOptions(), "foo", &v));
280 ASSERT_EQ("v5", v);
281 // Write something. If sequence number was not recovered properly,
282 // it will be hidden by an earlier write.
283 ASSERT_OK(db_->Put(WriteOptions(), "foo", "v6"));
284 ASSERT_OK(db_->Get(ReadOptions(), "foo", &v));
285 ASSERT_EQ("v6", v);
286 Reopen();
287 ASSERT_OK(db_->Get(ReadOptions(), "foo", &v));
288 ASSERT_EQ("v6", v);
289 }
290
291 TEST(CorruptionTest, CorruptedDescriptor) {
292 ASSERT_OK(db_->Put(WriteOptions(), "foo", "hello"));
293 DBImpl* dbi = reinterpret_cast<DBImpl*>(db_);
294 dbi->TEST_CompactMemTable();
295 dbi->TEST_CompactRange(0, nullptr, nullptr);
296
297 Corrupt(kDescriptorFile, 0, 1000);
298 Status s = TryReopen();
299 ASSERT_TRUE(!s.ok());
300
301 RepairDB();
302 Reopen();
303 std::string v;
304 ASSERT_OK(db_->Get(ReadOptions(), "foo", &v));
305 ASSERT_EQ("hello", v);
306 }
307
308 TEST(CorruptionTest, CompactionInputError) {
309 Build(10);
310 DBImpl* dbi = reinterpret_cast<DBImpl*>(db_);
311 dbi->TEST_CompactMemTable();
312 const int last = config::kMaxMemCompactLevel;
313 ASSERT_EQ(1, Property("leveldb.num-files-at-level" + NumberToString(last)));
314
315 Corrupt(kTableFile, 100, 1);
316 Check(5, 9);
317
318 // Force compactions by writing lots of values
319 Build(10000);
320 Check(10000, 10000);
321 }
322
323 TEST(CorruptionTest, CompactionInputErrorParanoid) {
324 options_.paranoid_checks = true;
325 options_.write_buffer_size = 512 << 10;
326 Reopen();
327 DBImpl* dbi = reinterpret_cast<DBImpl*>(db_);
328
329 // Make multiple inputs so we need to compact.
330 for (int i = 0; i < 2; i++) {
331 Build(10);
332 dbi->TEST_CompactMemTable();
333 Corrupt(kTableFile, 100, 1);
334 env_.SleepForMicroseconds(100000);
335 }
336 dbi->CompactRange(nullptr, nullptr);
337
338 // Write must fail because of corrupted table
339 std::string tmp1, tmp2;
340 Status s = db_->Put(WriteOptions(), Key(5, &tmp1), Value(5, &tmp2));
341 ASSERT_TRUE(!s.ok()) << "write did not fail in corrupted paranoid db";
342 }
343
344 TEST(CorruptionTest, UnrelatedKeys) {
345 Build(10);
346 DBImpl* dbi = reinterpret_cast<DBImpl*>(db_);
347 dbi->TEST_CompactMemTable();
348 Corrupt(kTableFile, 100, 1);
349
350 std::string tmp1, tmp2;
351 ASSERT_OK(db_->Put(WriteOptions(), Key(1000, &tmp1), Value(1000, &tmp2)));
352 std::string v;
353 ASSERT_OK(db_->Get(ReadOptions(), Key(1000, &tmp1), &v));
354 ASSERT_EQ(Value(1000, &tmp2).ToString(), v);
355 dbi->TEST_CompactMemTable();
356 ASSERT_OK(db_->Get(ReadOptions(), Key(1000, &tmp1), &v));
357 ASSERT_EQ(Value(1000, &tmp2).ToString(), v);
358 }
359
360 } // namespace leveldb
361
362 int main(int argc, char** argv) { return leveldb::test::RunAllTests(); }
363