OpenShot Library | libopenshot  0.3.3
CacheDisk.cpp
Go to the documentation of this file.
1 
9 // Copyright (c) 2008-2019 OpenShot Studios, LLC
10 //
11 // SPDX-License-Identifier: LGPL-3.0-or-later
12 
13 #include "CacheDisk.h"
14 #include "Exceptions.h"
15 #include "Frame.h"
16 #include "QtUtilities.h"
17 
18 #include <Qt>
19 #include <QString>
20 #include <QTextStream>
21 
22 using namespace std;
23 using namespace openshot;
24 
25 // Default constructor, no max bytes
26 CacheDisk::CacheDisk(std::string cache_path, std::string format, float quality, float scale) : CacheBase(0) {
27  // Set cache type name
28  cache_type = "CacheDisk";
29  range_version = 0;
30  needs_range_processing = false;
31  frame_size_bytes = 0;
32  image_format = format;
33  image_quality = quality;
34  image_scale = scale;
35  max_bytes = 0;
36 
37  // Init path directory
38  InitPath(cache_path);
39 }
40 
41 // Constructor that sets the max bytes to cache
42 CacheDisk::CacheDisk(std::string cache_path, std::string format, float quality, float scale, int64_t max_bytes) : CacheBase(max_bytes) {
43  // Set cache type name
44  cache_type = "CacheDisk";
45  range_version = 0;
46  needs_range_processing = false;
47  frame_size_bytes = 0;
48  image_format = format;
49  image_quality = quality;
50  image_scale = scale;
51 
52  // Init path directory
53  InitPath(cache_path);
54 }
55 
56 // Initialize cache directory
57 void CacheDisk::InitPath(std::string cache_path) {
58  QString qpath;
59 
60  if (!cache_path.empty()) {
61  // Init QDir with cache directory
62  qpath = QString(cache_path.c_str());
63 
64  } else {
65  // Init QDir with user's temp directory
66  qpath = QDir::tempPath() + QString("/preview-cache/");
67  }
68 
69  // Init QDir with cache directory
70  path = QDir(qpath);
71 
72  // Check if cache directory exists
73  if (!path.exists())
74  // Create
75  path.mkpath(qpath);
76 }
77 
78 // Default destructor
80 {
81  Clear();
82 
83  // remove mutex
84  delete cacheMutex;
85 }
86 
87 // Add a Frame to the cache
88 void CacheDisk::Add(std::shared_ptr<Frame> frame)
89 {
90  // Create a scoped lock, to protect the cache from multiple threads
91  const std::lock_guard<std::recursive_mutex> lock(*cacheMutex);
92  int64_t frame_number = frame->number;
93 
94  // Freshen frame if it already exists
95  if (frames.count(frame_number))
96  // Move frame to front of queue
97  MoveToFront(frame_number);
98 
99  else
100  {
101  // Add frame to queue and map
102  frames[frame_number] = frame_number;
103  frame_numbers.push_front(frame_number);
104  ordered_frame_numbers.push_back(frame_number);
105  needs_range_processing = true;
106 
107  // Save image to disk (if needed)
108  QString frame_path(path.path() + "/" + QString("%1.").arg(frame_number) + QString(image_format.c_str()).toLower());
109  frame->Save(frame_path.toStdString(), image_scale, image_format, image_quality);
110  if (frame_size_bytes == 0) {
111  // Get compressed size of frame image (to correctly apply max size against)
112  QFile image_file(frame_path);
113  frame_size_bytes = image_file.size();
114  }
115 
116  // Save audio data (if needed)
117  if (frame->has_audio_data) {
118  QString audio_path(path.path() + "/" + QString("%1").arg(frame_number) + ".audio");
119  QFile audio_file(audio_path);
120 
121  if (audio_file.open(QIODevice::WriteOnly)) {
122  QTextStream audio_stream(&audio_file);
123  audio_stream << frame->SampleRate() << Qt::endl;
124  audio_stream << frame->GetAudioChannelsCount() << Qt::endl;
125  audio_stream << frame->GetAudioSamplesCount() << Qt::endl;
126  audio_stream << frame->ChannelsLayout() << Qt::endl;
127 
128  // Loop through all samples
129  for (int channel = 0; channel < frame->GetAudioChannelsCount(); channel++)
130  {
131  // Get audio for this channel
132  float *samples = frame->GetAudioSamples(channel);
133  for (int sample = 0; sample < frame->GetAudioSamplesCount(); sample++)
134  audio_stream << samples[sample] << Qt::endl;
135  }
136 
137  }
138 
139  }
140 
141  // Clean up old frames
142  CleanUp();
143  }
144 }
145 
146 // Check if frame is already contained in cache
147 bool CacheDisk::Contains(int64_t frame_number) {
148  if (frames.count(frame_number) > 0) {
149  return true;
150  } else {
151  return false;
152  }
153 }
154 
155 // Get a frame from the cache (or NULL shared_ptr if no frame is found)
156 std::shared_ptr<Frame> CacheDisk::GetFrame(int64_t frame_number)
157 {
158  // Create a scoped lock, to protect the cache from multiple threads
159  const std::lock_guard<std::recursive_mutex> lock(*cacheMutex);
160 
161  // Does frame exists in cache?
162  if (frames.count(frame_number)) {
163  // Does frame exist on disk
164  QString frame_path(path.path() + "/" + QString("%1.").arg(frame_number) + QString(image_format.c_str()).toLower());
165  if (path.exists(frame_path)) {
166 
167  // Load image file
168  auto image = std::make_shared<QImage>();
169  image->load(frame_path);
170 
171  // Set pixel formatimage->
172  image = std::make_shared<QImage>(image->convertToFormat(QImage::Format_RGBA8888_Premultiplied));
173 
174  // Create frame object
175  auto frame = std::make_shared<Frame>();
176  frame->number = frame_number;
177  frame->AddImage(image);
178 
179  // Get audio data (if found)
180  QString audio_path(path.path() + "/" + QString("%1").arg(frame_number) + ".audio");
181  QFile audio_file(audio_path);
182  if (audio_file.exists()) {
183  // Open audio file
184  QTextStream in(&audio_file);
185  if (audio_file.open(QIODevice::ReadOnly)) {
186  int sample_rate = in.readLine().toInt();
187  int channels = in.readLine().toInt();
188  int sample_count = in.readLine().toInt();
189  int channel_layout = in.readLine().toInt();
190 
191  // Set basic audio properties
192  frame->ResizeAudio(channels, sample_count, sample_rate, (ChannelLayout) channel_layout);
193 
194  // Loop through audio samples and add to frame
195  int current_channel = 0;
196  int current_sample = 0;
197  float *channel_samples = new float[sample_count];
198  while (!in.atEnd()) {
199  // Add sample to channel array
200  channel_samples[current_sample] = in.readLine().toFloat();
201  current_sample++;
202 
203  if (current_sample == sample_count) {
204  // Add audio to frame
205  frame->AddAudio(true, current_channel, 0, channel_samples, sample_count, 1.0);
206 
207  // Increment channel, and reset sample position
208  current_channel++;
209  current_sample = 0;
210  }
211 
212  }
213  }
214  }
215 
216  // return the Frame object
217  return frame;
218  }
219  }
220 
221  // no Frame found
222  return std::shared_ptr<Frame>();
223 }
224 
225 // @brief Get an array of all Frames
226 std::vector<std::shared_ptr<openshot::Frame>> CacheDisk::GetFrames()
227 {
228  // Create a scoped lock, to protect the cache from multiple threads
229  const std::lock_guard<std::recursive_mutex> lock(*cacheMutex);
230 
231  std::vector<std::shared_ptr<openshot::Frame>> all_frames;
232  std::vector<int64_t>::iterator itr_ordered;
233  for(itr_ordered = ordered_frame_numbers.begin(); itr_ordered != ordered_frame_numbers.end(); ++itr_ordered)
234  {
235  int64_t frame_number = *itr_ordered;
236  all_frames.push_back(GetFrame(frame_number));
237  }
238 
239  return all_frames;
240 }
241 
242 // Get the smallest frame number (or NULL shared_ptr if no frame is found)
243 std::shared_ptr<Frame> CacheDisk::GetSmallestFrame()
244 {
245  // Create a scoped lock, to protect the cache from multiple threads
246  const std::lock_guard<std::recursive_mutex> lock(*cacheMutex);
247 
248  // Loop through frame numbers
249  std::deque<int64_t>::iterator itr;
250  int64_t smallest_frame = -1;
251  for(itr = frame_numbers.begin(); itr != frame_numbers.end(); ++itr)
252  {
253  if (*itr < smallest_frame || smallest_frame == -1)
254  smallest_frame = *itr;
255  }
256 
257  // Return frame (if any)
258  if (smallest_frame != -1) {
259  return GetFrame(smallest_frame);
260  } else {
261  return NULL;
262  }
263 }
264 
265 // Gets the maximum bytes value
267 {
268  // Create a scoped lock, to protect the cache from multiple threads
269  const std::lock_guard<std::recursive_mutex> lock(*cacheMutex);
270 
271  int64_t total_bytes = 0;
272 
273  // Loop through frames, and calculate total bytes
274  std::deque<int64_t>::reverse_iterator itr;
275  for(itr = frame_numbers.rbegin(); itr != frame_numbers.rend(); ++itr)
276  total_bytes += frame_size_bytes;
277 
278  return total_bytes;
279 }
280 
281 // Remove a specific frame
282 void CacheDisk::Remove(int64_t frame_number)
283 {
284  Remove(frame_number, frame_number);
285 }
286 
287 // Remove range of frames
288 void CacheDisk::Remove(int64_t start_frame_number, int64_t end_frame_number)
289 {
290  // Create a scoped lock, to protect the cache from multiple threads
291  const std::lock_guard<std::recursive_mutex> lock(*cacheMutex);
292 
293  // Loop through frame numbers
294  std::deque<int64_t>::iterator itr;
295  for(itr = frame_numbers.begin(); itr != frame_numbers.end();)
296  {
297  //deque<int64_t>::iterator current = itr++;
298  if (*itr >= start_frame_number && *itr <= end_frame_number)
299  {
300  // erase frame number
301  itr = frame_numbers.erase(itr);
302  } else
303  itr++;
304  }
305 
306  // Loop through ordered frame numbers
307  std::vector<int64_t>::iterator itr_ordered;
308  for(itr_ordered = ordered_frame_numbers.begin(); itr_ordered != ordered_frame_numbers.end();)
309  {
310  if (*itr_ordered >= start_frame_number && *itr_ordered <= end_frame_number)
311  {
312  // erase frame number
313  frames.erase(*itr_ordered);
314 
315  // Remove the image file (if it exists)
316  QString frame_path(path.path() + "/" + QString("%1.").arg(*itr_ordered) + QString(image_format.c_str()).toLower());
317  QFile image_file(frame_path);
318  if (image_file.exists())
319  image_file.remove();
320 
321  // Remove audio file (if it exists)
322  QString audio_path(path.path() + "/" + QString("%1").arg(*itr_ordered) + ".audio");
323  QFile audio_file(audio_path);
324  if (audio_file.exists())
325  audio_file.remove();
326 
327  itr_ordered = ordered_frame_numbers.erase(itr_ordered);
328  } else
329  itr_ordered++;
330  }
331 
332  // Needs range processing (since cache has changed)
333  needs_range_processing = true;
334 }
335 
336 // Move frame to front of queue (so it lasts longer)
337 void CacheDisk::MoveToFront(int64_t frame_number)
338 {
339  // Does frame exists in cache?
340  if (frames.count(frame_number))
341  {
342  // Create a scoped lock, to protect the cache from multiple threads
343  const std::lock_guard<std::recursive_mutex> lock(*cacheMutex);
344 
345  // Loop through frame numbers
346  std::deque<int64_t>::iterator itr;
347  for(itr = frame_numbers.begin(); itr != frame_numbers.end(); ++itr)
348  {
349  if (*itr == frame_number)
350  {
351  // erase frame number
352  frame_numbers.erase(itr);
353 
354  // add frame number to 'front' of queue
355  frame_numbers.push_front(frame_number);
356  break;
357  }
358  }
359  }
360 }
361 
362 // Clear the cache of all frames
364 {
365  // Create a scoped lock, to protect the cache from multiple threads
366  const std::lock_guard<std::recursive_mutex> lock(*cacheMutex);
367 
368  // Clear all containers
369  frames.clear();
370  frame_numbers.clear();
371  frame_numbers.shrink_to_fit();
372  ordered_frame_numbers.clear();
373  ordered_frame_numbers.shrink_to_fit();
374  needs_range_processing = true;
375  frame_size_bytes = 0;
376 
377  // Delete cache directory, and recreate it
378  QString current_path = path.path();
379  path.removeRecursively();
380 
381  // Re-init folder
382  InitPath(current_path.toStdString());
383 }
384 
385 // Count the frames in the queue
387 {
388  // Create a scoped lock, to protect the cache from multiple threads
389  const std::lock_guard<std::recursive_mutex> lock(*cacheMutex);
390 
391  // Return the number of frames in the cache
392  return frames.size();
393 }
394 
395 // Clean up cached frames that exceed the number in our max_bytes variable
396 void CacheDisk::CleanUp()
397 {
398  // Do we auto clean up?
399  if (max_bytes > 0)
400  {
401  // Create a scoped lock, to protect the cache from multiple threads
402  const std::lock_guard<std::recursive_mutex> lock(*cacheMutex);
403 
404  while (GetBytes() > max_bytes && frame_numbers.size() > 20)
405  {
406  // Get the oldest frame number.
407  int64_t frame_to_remove = frame_numbers.back();
408 
409  // Remove frame_number and frame
410  Remove(frame_to_remove);
411  }
412  }
413 }
414 
415 // Generate JSON string of this object
416 std::string CacheDisk::Json() {
417 
418  // Return formatted string
419  return JsonValue().toStyledString();
420 }
421 
422 // Generate Json::Value for this object
423 Json::Value CacheDisk::JsonValue() {
424 
425  // Process range data (if anything has changed)
426  CalculateRanges();
427 
428  // Create root json object
429  Json::Value root = CacheBase::JsonValue(); // get parent properties
430  root["type"] = cache_type;
431  root["path"] = path.path().toStdString();
432 
433  Json::Value version;
434  std::stringstream range_version_str;
435  range_version_str << range_version;
436  root["version"] = range_version_str.str();
437 
438  // Parse and append range data (if any)
439  // Parse and append range data (if any)
440  try {
441  const Json::Value ranges = openshot::stringToJson(json_ranges);
442  root["ranges"] = ranges;
443  } catch (...) { }
444 
445  // return JsonValue
446  return root;
447 }
448 
449 // Load JSON string into this object
450 void CacheDisk::SetJson(const std::string value) {
451 
452  // Parse JSON string into JSON objects
453  try
454  {
455  const Json::Value root = openshot::stringToJson(value);
456  // Set all values that match
457  SetJsonValue(root);
458  }
459  catch (const std::exception& e)
460  {
461  // Error parsing JSON (or missing keys)
462  throw InvalidJSON("JSON is invalid (missing keys or invalid data types)");
463  }
464 }
465 
466 // Load Json::Value into this object
467 void CacheDisk::SetJsonValue(const Json::Value root) {
468 
469  // Close timeline before we do anything (this also removes all open and closing clips)
470  Clear();
471 
472  // Set parent data
474 
475  if (!root["type"].isNull())
476  cache_type = root["type"].asString();
477  if (!root["path"].isNull())
478  // Update duration of timeline
479  InitPath(root["path"].asString());
480 }
openshot::stringToJson
const Json::Value stringToJson(const std::string value)
Definition: Json.cpp:16
openshot::CacheBase::needs_range_processing
bool needs_range_processing
Something has changed, and the range data needs to be re-calculated.
Definition: CacheBase.h:40
openshot::CacheBase::max_bytes
int64_t max_bytes
This is the max number of bytes to cache (0 = no limit)
Definition: CacheBase.h:38
openshot::CacheBase::ordered_frame_numbers
std::vector< int64_t > ordered_frame_numbers
Ordered list of frame numbers used by cache.
Definition: CacheBase.h:42
openshot
This namespace is the default namespace for all code in the openshot library.
Definition: Compressor.h:28
openshot::CacheDisk::Contains
bool Contains(int64_t frame_number)
Check if frame is already contained in cache.
Definition: CacheDisk.cpp:147
openshot::CacheBase::json_ranges
std::string json_ranges
JSON ranges of frame numbers.
Definition: CacheBase.h:41
CacheDisk.h
Header file for CacheDisk class.
QtUtilities.h
Header file for QtUtilities (compatibiity overlay)
openshot::CacheDisk::Remove
void Remove(int64_t frame_number)
Remove a specific frame.
Definition: CacheDisk.cpp:282
openshot::CacheDisk::Clear
void Clear()
Clear the cache of all frames.
Definition: CacheDisk.cpp:363
openshot::CacheBase
All cache managers in libopenshot are based on this CacheBase class.
Definition: CacheBase.h:34
openshot::CacheBase::SetJsonValue
virtual void SetJsonValue(const Json::Value root)=0
Load Json::Value into this object.
Definition: CacheBase.cpp:115
openshot::CacheDisk::SetJsonValue
void SetJsonValue(const Json::Value root)
Load Json::Value into this object.
Definition: CacheDisk.cpp:467
openshot::CacheDisk::Count
int64_t Count()
Count the frames in the queue.
Definition: CacheDisk.cpp:386
openshot::InvalidJSON
Exception for invalid JSON.
Definition: Exceptions.h:217
openshot::CacheDisk::Json
std::string Json()
Generate JSON string of this object.
Definition: CacheDisk.cpp:416
openshot::CacheDisk::CacheDisk
CacheDisk(std::string cache_path, std::string format, float quality, float scale)
Default constructor, no max bytes.
Definition: CacheDisk.cpp:26
openshot::CacheDisk::GetFrame
std::shared_ptr< openshot::Frame > GetFrame(int64_t frame_number)
Get a frame from the cache.
Definition: CacheDisk.cpp:156
openshot::CacheBase::JsonValue
virtual Json::Value JsonValue()=0
Generate Json::Value for this object.
Definition: CacheBase.cpp:102
openshot::CacheBase::CalculateRanges
void CalculateRanges()
Calculate ranges of frames.
Definition: CacheBase.cpp:38
openshot::CacheDisk::GetBytes
int64_t GetBytes()
Gets the maximum bytes value.
Definition: CacheDisk.cpp:266
Frame.h
Header file for Frame class.
openshot::CacheBase::cache_type
std::string cache_type
This is a friendly type name of the derived cache instance.
Definition: CacheBase.h:37
openshot::CacheDisk::GetFrames
std::vector< std::shared_ptr< openshot::Frame > > GetFrames()
Get an array of all Frames.
Definition: CacheDisk.cpp:226
openshot::CacheDisk::JsonValue
Json::Value JsonValue()
Generate Json::Value for this object.
Definition: CacheDisk.cpp:423
openshot::CacheDisk::GetSmallestFrame
std::shared_ptr< openshot::Frame > GetSmallestFrame()
Get the smallest frame number.
Definition: CacheDisk.cpp:243
openshot::CacheDisk::SetJson
void SetJson(const std::string value)
Load JSON string into this object.
Definition: CacheDisk.cpp:450
openshot::CacheDisk::Add
void Add(std::shared_ptr< openshot::Frame > frame)
Add a Frame to the cache.
Definition: CacheDisk.cpp:88
openshot::ChannelLayout
ChannelLayout
This enumeration determines the audio channel layout (such as stereo, mono, 5 point surround,...
Definition: ChannelLayouts.h:28
openshot::CacheDisk::MoveToFront
void MoveToFront(int64_t frame_number)
Move frame to front of queue (so it lasts longer)
Definition: CacheDisk.cpp:337
openshot::CacheDisk::~CacheDisk
~CacheDisk()
Definition: CacheDisk.cpp:79
openshot::CacheBase::range_version
int64_t range_version
The version of the JSON range data (incremented with each change)
Definition: CacheBase.h:44
Exceptions.h
Header file for all Exception classes.
openshot::CacheBase::cacheMutex
std::recursive_mutex * cacheMutex
Mutex for multiple threads.
Definition: CacheBase.h:47