summaryrefslogtreecommitdiffstats
path: root/updater
diff options
context:
space:
mode:
Diffstat (limited to 'updater')
-rw-r--r--updater/Android.mk1
-rw-r--r--updater/blockimg.cpp154
-rw-r--r--updater/include/updater/rangeset.h116
-rw-r--r--updater/install.cpp31
4 files changed, 243 insertions, 59 deletions
diff --git a/updater/Android.mk b/updater/Android.mk
index a113fe86c..86dc48e30 100644
--- a/updater/Android.mk
+++ b/updater/Android.mk
@@ -47,6 +47,7 @@ updater_common_static_libraries := \
libcrypto_utils \
libcutils \
libtune2fs \
+ libbrotli \
$(tune2fs_static_libraries)
# libupdater (static library)
diff --git a/updater/blockimg.cpp b/updater/blockimg.cpp
index df366b0b8..a0b9ad233 100644
--- a/updater/blockimg.cpp
+++ b/updater/blockimg.cpp
@@ -44,6 +44,7 @@
#include <android-base/strings.h>
#include <android-base/unique_fd.h>
#include <applypatch/applypatch.h>
+#include <brotli/decode.h>
#include <openssl/sha.h>
#include <private/android_filesystem_config.h>
#include <ziparchive/zip_archive.h>
@@ -149,7 +150,11 @@ static void allocate(size_t size, std::vector<uint8_t>& buffer) {
class RangeSinkWriter {
public:
RangeSinkWriter(int fd, const RangeSet& tgt)
- : fd_(fd), tgt_(tgt), next_range_(0), current_range_left_(0), bytes_written_(0) {
+ : fd_(fd),
+ tgt_(tgt),
+ next_range_(0),
+ current_range_left_(0),
+ bytes_written_(0) {
CHECK_NE(tgt.size(), static_cast<size_t>(0));
};
@@ -157,6 +162,11 @@ class RangeSinkWriter {
return next_range_ == tgt_.size() && current_range_left_ == 0;
}
+ size_t AvailableSpace() const {
+ return tgt_.blocks() * BLOCKSIZE - bytes_written_;
+ }
+
+ // Return number of bytes written; and 0 indicates a writing failure.
size_t Write(const uint8_t* data, size_t size) {
if (Finished()) {
LOG(ERROR) << "range sink write overrun; can't write " << size << " bytes";
@@ -166,23 +176,8 @@ class RangeSinkWriter {
size_t written = 0;
while (size > 0) {
// Move to the next range as needed.
- if (current_range_left_ == 0) {
- if (next_range_ < tgt_.size()) {
- const Range& range = tgt_[next_range_];
- off64_t offset = static_cast<off64_t>(range.first) * BLOCKSIZE;
- current_range_left_ = (range.second - range.first) * BLOCKSIZE;
- next_range_++;
- if (!discard_blocks(fd_, offset, current_range_left_)) {
- break;
- }
-
- if (!check_lseek(fd_, offset, SEEK_SET)) {
- break;
- }
- } else {
- // We can't write any more; return how many bytes have been written so far.
- break;
- }
+ if (!SeekToOutputRange()) {
+ break;
}
size_t write_now = size;
@@ -210,9 +205,35 @@ class RangeSinkWriter {
}
private:
- // The input data.
+ // Set up the output cursor, move to next range if needed.
+ bool SeekToOutputRange() {
+ // We haven't finished the current range yet.
+ if (current_range_left_ != 0) {
+ return true;
+ }
+ // We can't write any more; let the write function return how many bytes have been written
+ // so far.
+ if (next_range_ >= tgt_.size()) {
+ return false;
+ }
+
+ const Range& range = tgt_[next_range_];
+ off64_t offset = static_cast<off64_t>(range.first) * BLOCKSIZE;
+ current_range_left_ = (range.second - range.first) * BLOCKSIZE;
+ next_range_++;
+
+ if (!discard_blocks(fd_, offset, current_range_left_)) {
+ return false;
+ }
+ if (!check_lseek(fd_, offset, SEEK_SET)) {
+ return false;
+ }
+ return true;
+ }
+
+ // The output file descriptor.
int fd_;
- // The destination for the data.
+ // The destination ranges for the data.
const RangeSet& tgt_;
// The next range that we should write to.
size_t next_range_;
@@ -243,8 +264,10 @@ class RangeSinkWriter {
struct NewThreadInfo {
ZipArchiveHandle za;
ZipEntry entry;
+ bool brotli_compressed;
- RangeSinkWriter* writer;
+ std::unique_ptr<RangeSinkWriter> writer;
+ BrotliDecoderState* brotli_decoder_state;
bool receiver_available;
pthread_mutex_t mu;
@@ -264,9 +287,14 @@ static bool receive_new_data(const uint8_t* data, size_t size, void* cookie) {
// At this point nti->writer is set, and we own it. The main thread is waiting for it to
// disappear from nti.
- size_t written = nti->writer->Write(data, size);
- data += written;
- size -= written;
+ size_t write_now = std::min(size, nti->writer->AvailableSpace());
+ if (nti->writer->Write(data, write_now) != write_now) {
+ LOG(ERROR) << "Failed to write " << write_now << " bytes.";
+ return false;
+ }
+
+ data += write_now;
+ size -= write_now;
if (nti->writer->Finished()) {
// We have written all the bytes desired by this writer.
@@ -281,10 +309,72 @@ static bool receive_new_data(const uint8_t* data, size_t size, void* cookie) {
return true;
}
-static void* unzip_new_data(void* cookie) {
+static bool receive_brotli_new_data(const uint8_t* data, size_t size, void* cookie) {
NewThreadInfo* nti = static_cast<NewThreadInfo*>(cookie);
- ProcessZipEntryContents(nti->za, &nti->entry, receive_new_data, nti);
+ while (size > 0 || BrotliDecoderHasMoreOutput(nti->brotli_decoder_state)) {
+ // Wait for nti->writer to be non-null, indicating some of this data is wanted.
+ pthread_mutex_lock(&nti->mu);
+ while (nti->writer == nullptr) {
+ pthread_cond_wait(&nti->cv, &nti->mu);
+ }
+ pthread_mutex_unlock(&nti->mu);
+
+ // At this point nti->writer is set, and we own it. The main thread is waiting for it to
+ // disappear from nti.
+
+ size_t buffer_size = std::min<size_t>(32768, nti->writer->AvailableSpace());
+ if (buffer_size == 0) {
+ LOG(ERROR) << "No space left in output range";
+ return false;
+ }
+ uint8_t buffer[buffer_size];
+ size_t available_in = size;
+ size_t available_out = buffer_size;
+ uint8_t* next_out = buffer;
+
+ // The brotli decoder will update |data|, |available_in|, |next_out| and |available_out|.
+ BrotliDecoderResult result = BrotliDecoderDecompressStream(
+ nti->brotli_decoder_state, &available_in, &data, &available_out, &next_out, nullptr);
+
+ if (result == BROTLI_DECODER_RESULT_ERROR) {
+ LOG(ERROR) << "Decompression failed with "
+ << BrotliDecoderErrorString(BrotliDecoderGetErrorCode(nti->brotli_decoder_state));
+ return false;
+ }
+
+ LOG(DEBUG) << "bytes to write: " << buffer_size - available_out << ", bytes consumed "
+ << size - available_in << ", decoder status " << result;
+
+ size_t write_now = buffer_size - available_out;
+ if (nti->writer->Write(buffer, write_now) != write_now) {
+ LOG(ERROR) << "Failed to write " << write_now << " bytes.";
+ return false;
+ }
+
+ // Update the remaining size. The input data ptr is already updated by brotli decoder function.
+ size = available_in;
+
+ if (nti->writer->Finished()) {
+ // We have written all the bytes desired by this writer.
+
+ pthread_mutex_lock(&nti->mu);
+ nti->writer = nullptr;
+ pthread_cond_broadcast(&nti->cv);
+ pthread_mutex_unlock(&nti->mu);
+ }
+ }
+
+ return true;
+}
+
+static void* unzip_new_data(void* cookie) {
+ NewThreadInfo* nti = static_cast<NewThreadInfo*>(cookie);
+ if (nti->brotli_compressed) {
+ ProcessZipEntryContents(nti->za, &nti->entry, receive_brotli_new_data, nti);
+ } else {
+ ProcessZipEntryContents(nti->za, &nti->entry, receive_new_data, nti);
+ }
pthread_mutex_lock(&nti->mu);
nti->receiver_available = false;
if (nti->writer != nullptr) {
@@ -1142,9 +1232,8 @@ static int PerformCommandNew(CommandParameters& params) {
if (params.canwrite) {
LOG(INFO) << " writing " << tgt.blocks() << " blocks of new data";
- RangeSinkWriter writer(params.fd, tgt);
pthread_mutex_lock(&params.nti.mu);
- params.nti.writer = &writer;
+ params.nti.writer = std::make_unique<RangeSinkWriter>(params.fd, tgt);
pthread_cond_broadcast(&params.nti.cv);
while (params.nti.writer != nullptr) {
@@ -1384,6 +1473,11 @@ static Value* PerformBlockImageUpdate(const char* name, State* state,
if (params.canwrite) {
params.nti.za = za;
params.nti.entry = new_entry;
+ params.nti.brotli_compressed = android::base::EndsWith(new_data_fn->data, ".br");
+ if (params.nti.brotli_compressed) {
+ // Initialize brotli decoder state.
+ params.nti.brotli_decoder_state = BrotliDecoderCreateInstance(nullptr, nullptr, nullptr);
+ }
params.nti.receiver_available = true;
pthread_mutex_init(&params.nti.mu, nullptr);
@@ -1526,6 +1620,10 @@ pbiudone:
}
// params.fd will be automatically closed because it's a unique_fd.
+ if (params.nti.brotli_decoder_state != nullptr) {
+ BrotliDecoderDestroyInstance(params.nti.brotli_decoder_state);
+ }
+
// Only delete the stash if the update cannot be resumed, or it's a verification run and we
// created the stash.
if (params.isunresumable || (!params.canwrite && params.createdstash)) {
diff --git a/updater/include/updater/rangeset.h b/updater/include/updater/rangeset.h
index fad038043..b67c98724 100644
--- a/updater/include/updater/rangeset.h
+++ b/updater/include/updater/rangeset.h
@@ -24,6 +24,7 @@
#include <android-base/logging.h>
#include <android-base/parseint.h>
+#include <android-base/stringprintf.h>
#include <android-base/strings.h>
using Range = std::pair<size_t, size_t>;
@@ -74,6 +75,18 @@ class RangeSet {
return RangeSet(std::move(pairs));
}
+ std::string ToString() const {
+ if (ranges_.empty()) {
+ return "";
+ }
+ std::string result = std::to_string(ranges_.size() * 2);
+ for (const auto& r : ranges_) {
+ result += android::base::StringPrintf(",%zu,%zu", r.first, r.second);
+ }
+
+ return result;
+ }
+
// Get the block number for the i-th (starting from 0) block in the RangeSet.
size_t GetBlockNumber(size_t idx) const {
CHECK_LT(idx, blocks_) << "Out of bound index " << idx << " (total blocks: " << blocks_ << ")";
@@ -157,8 +170,109 @@ class RangeSet {
return ranges_ != other.ranges_;
}
- private:
+ protected:
// Actual limit for each value and the total number are both INT_MAX.
std::vector<Range> ranges_;
size_t blocks_;
};
+
+static constexpr size_t kBlockSize = 4096;
+
+// The class is a sorted version of a RangeSet; and it's useful in imgdiff to split the input
+// files when we're handling large zip files. Specifically, we can treat the input file as a
+// continuous RangeSet (i.e. RangeSet("0-99") for a 100 blocks file); and break it down into
+// several smaller chunks based on the zip entries.
+
+// For example, [source: 0-99] can be split into
+// [split_src1: 10-29]; [split_src2: 40-49, 60-69]; [split_src3: 70-89]
+// Here "10-29" simply means block 10th to block 29th with respect to the original input file.
+// Also, note that the split sources should be mutual exclusive, but they don't need to cover
+// every block in the original source.
+class SortedRangeSet : public RangeSet {
+ public:
+ SortedRangeSet() {}
+
+ // Ranges in the the set should be mutually exclusive; and they're sorted by the start block.
+ explicit SortedRangeSet(std::vector<Range>&& pairs) : RangeSet(std::move(pairs)) {
+ std::sort(ranges_.begin(), ranges_.end());
+ }
+
+ void Insert(const Range& to_insert) {
+ SortedRangeSet rs({ to_insert });
+ Insert(rs);
+ }
+
+ // Insert the input SortedRangeSet; keep the ranges sorted and merge the overlap ranges.
+ void Insert(const SortedRangeSet& rs) {
+ if (rs.size() == 0) {
+ return;
+ }
+ // Merge and sort the two RangeSets.
+ std::vector<Range> temp = std::move(ranges_);
+ std::copy(rs.begin(), rs.end(), std::back_inserter(temp));
+ std::sort(temp.begin(), temp.end());
+
+ Clear();
+ // Trim overlaps and insert the result back to ranges_.
+ Range to_insert = temp.front();
+ for (auto it = temp.cbegin() + 1; it != temp.cend(); it++) {
+ if (it->first <= to_insert.second) {
+ to_insert.second = std::max(to_insert.second, it->second);
+ } else {
+ ranges_.push_back(to_insert);
+ blocks_ += (to_insert.second - to_insert.first);
+ to_insert = *it;
+ }
+ }
+ ranges_.push_back(to_insert);
+ blocks_ += (to_insert.second - to_insert.first);
+ }
+
+ void Clear() {
+ blocks_ = 0;
+ ranges_.clear();
+ }
+
+ using RangeSet::Overlaps;
+ bool Overlaps(size_t start, size_t len) const {
+ RangeSet rs({ { start / kBlockSize, (start + len - 1) / kBlockSize + 1 } });
+ return Overlaps(rs);
+ }
+
+ // Compute the block range the file occupies, and insert that range.
+ void Insert(size_t start, size_t len) {
+ Range to_insert{ start / kBlockSize, (start + len - 1) / kBlockSize + 1 };
+ Insert(to_insert);
+ }
+
+ // Given an offset of the file, checks if the corresponding block (by considering the file as
+ // 0-based continuous block ranges) is covered by the SortedRangeSet. If so, returns the offset
+ // within this SortedRangeSet.
+ //
+ // For example, the 4106-th byte of a file is from block 1, assuming a block size of 4096-byte.
+ // The mapped offset within a SortedRangeSet("1-9 15-19") is 10.
+ //
+ // An offset of 65546 falls into the 16-th block in a file. Block 16 is contained as the 10-th
+ // item in SortedRangeSet("1-9 15-19"). So its data can be found at offset 40970 (i.e. 4096 * 10
+ // + 10) in a range represented by this SortedRangeSet.
+ size_t GetOffsetInRangeSet(size_t old_offset) const {
+ size_t old_block_start = old_offset / kBlockSize;
+ size_t new_block_start = 0;
+ for (const auto& range : ranges_) {
+ // Find the index of old_block_start.
+ if (old_block_start >= range.second) {
+ new_block_start += (range.second - range.first);
+ } else if (old_block_start >= range.first) {
+ new_block_start += (old_block_start - range.first);
+ return (new_block_start * kBlockSize + old_offset % kBlockSize);
+ } else {
+ CHECK(false) <<"block_start " << old_block_start << " is missing between two ranges: "
+ << this->ToString();
+ return 0;
+ }
+ }
+ CHECK(false) <<"block_start " << old_block_start << " exceeds the limit of current RangeSet: "
+ << this->ToString();
+ return 0;
+ }
+}; \ No newline at end of file
diff --git a/updater/install.cpp b/updater/install.cpp
index c9a3a0799..8e54c2e75 100644
--- a/updater/install.cpp
+++ b/updater/install.cpp
@@ -95,34 +95,6 @@ void uiPrintf(State* _Nonnull state, const char* _Nonnull format, ...) {
uiPrint(state, error_msg);
}
-static bool is_dir(const std::string& dirpath) {
- struct stat st;
- return stat(dirpath.c_str(), &st) == 0 && S_ISDIR(st.st_mode);
-}
-
-// Create all parent directories of name, if necessary.
-static bool make_parents(const std::string& name) {
- size_t prev_end = 0;
- while (prev_end < name.size()) {
- size_t next_end = name.find('/', prev_end + 1);
- if (next_end == std::string::npos) {
- break;
- }
- std::string dir_path = name.substr(0, next_end);
- if (!is_dir(dir_path)) {
- int result = mkdir(dir_path.c_str(), 0700);
- if (result != 0) {
- PLOG(ERROR) << "failed to mkdir " << dir_path << " when make parents for " << name;
- return false;
- }
-
- LOG(INFO) << "created [" << dir_path << "]";
- }
- prev_end = next_end;
- }
- return true;
-}
-
// mount(fs_type, partition_type, location, mount_point)
// mount(fs_type, partition_type, location, mount_point, mount_options)
@@ -322,8 +294,7 @@ Value* FormatFn(const char* name, State* state, const std::vector<std::unique_pt
return StringValue(location);
}
- const char* e2fsdroid_argv[] = { "/sbin/e2fsdroid_static", "-e", "-S",
- "/file_contexts", "-a", mount_point.c_str(),
+ const char* e2fsdroid_argv[] = { "/sbin/e2fsdroid_static", "-e", "-a", mount_point.c_str(),
location.c_str(), nullptr };
status = exec_cmd(e2fsdroid_argv[0], const_cast<char**>(e2fsdroid_argv));
if (status != 0) {
generated by cgit v1.2.3 (git 2.25.1) at 2024-07-09 07:37:41 +0200