OR-Tools  8.2
file_util.cc
Go to the documentation of this file.
1 // Copyright 2010-2018 Google LLC
2 // Licensed under the Apache License, Version 2.0 (the "License");
3 // you may not use this file except in compliance with the License.
4 // You may obtain a copy of the License at
5 //
6 // http://www.apache.org/licenses/LICENSE-2.0
7 //
8 // Unless required by applicable law or agreed to in writing, software
9 // distributed under the License is distributed on an "AS IS" BASIS,
10 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11 // See the License for the specific language governing permissions and
12 // limitations under the License.
13 
14 #include "ortools/util/file_util.h"
15 
16 #include "absl/status/statusor.h"
17 #include "absl/strings/str_cat.h"
18 #include "google/protobuf/descriptor.h"
19 #include "google/protobuf/io/zero_copy_stream_impl_lite.h"
20 #include "google/protobuf/message.h"
21 #include "google/protobuf/text_format.h"
22 #include "google/protobuf/util/json_util.h"
23 #include "ortools/base/file.h"
25 #include "ortools/base/logging.h"
27 
28 namespace operations_research {
29 
30 using ::google::protobuf::TextFormat;
31 
32 absl::StatusOr<std::string> ReadFileToString(absl::string_view filename) {
33  std::string contents;
34  RETURN_IF_ERROR(file::GetContents(filename, &contents, file::Defaults()));
35  // Try decompressing it.
36  {
37  std::string uncompressed;
38  if (GunzipString(contents, &uncompressed)) contents.swap(uncompressed);
39  }
40  return contents;
41 }
42 
43 bool ReadFileToProto(absl::string_view filename,
44  google::protobuf::Message* proto) {
45  std::string data;
46  CHECK_OK(file::GetContents(filename, &data, file::Defaults()));
47  // Try decompressing it.
48  {
49  std::string uncompressed;
50  if (GunzipString(data, &uncompressed)) {
51  VLOG(1) << "ReadFileToProto(): input is gzipped";
52  data.swap(uncompressed);
53  }
54  }
55  // Try binary format first, then text format, then JSON, then proto3 JSON,
56  // then give up.
57  // For some of those, like binary format and proto3 JSON, we perform
58  // additional checks to verify that we have the right proto: it can happen
59  // to try to read a proto of type Foo as a proto of type Bar, by mistake, and
60  // we'd rather have this function fail rather than silently accept it, because
61  // the proto parser is too lenient with unknown fields.
62  // We don't require ByteSizeLong(parsed) == input.size(), because it may be
63  // the case that the proto version changed and some fields are dropped.
64  // We just fail when the difference is too large.
65  constexpr double kMaxBinaryProtoParseShrinkFactor = 2;
66  if (proto->ParseFromString(data)) {
67  // NOTE(user): When using ParseFromString() from a generic
68  // google::protobuf::Message, like we do here, all fields are stored, even
69  // if their are unknown in the underlying proto type. Unless we explicitly
70  // discard those 'unknown fields' here, our call to ByteSizeLong() will
71  // still count the unknown payload.
72  proto->DiscardUnknownFields();
73  if (proto->ByteSizeLong() <
74  data.size() / kMaxBinaryProtoParseShrinkFactor) {
75  VLOG(1) << "ReadFileToProto(): input may be a binary proto, but of a "
76  "different proto";
77  } else {
78  VLOG(1) << "ReadFileToProto(): input seems to be a binary proto";
79  return true;
80  }
81  }
82  if (google::protobuf::TextFormat::ParseFromString(data, proto)) {
83  VLOG(1) << "ReadFileToProto(): input is a text proto";
84  return true;
85  }
86  if (google::protobuf::util::JsonStringToMessage(
87  data, proto, google::protobuf::util::JsonParseOptions())
88  .ok()) {
89  constexpr int kMaxJsonToBinaryShrinkFactor = 30;
90  if (proto->ByteSizeLong() < data.size() / kMaxJsonToBinaryShrinkFactor) {
91  VLOG(1) << "ReadFileToProto(): input is probably JSON, but probably not"
92  " of the right proto";
93  } else {
94  VLOG(1) << "ReadFileToProto(): input is a proto JSON";
95  return true;
96  }
97  }
98  LOG(WARNING) << "Could not parse protocol buffer";
99  return false;
100 }
101 
102 bool WriteProtoToFile(absl::string_view filename,
103  const google::protobuf::Message& proto,
104  ProtoWriteFormat proto_write_format, bool gzipped,
105  bool append_extension_to_file_name) {
106  std::string file_type_suffix;
107  std::string output_string;
108  google::protobuf::io::StringOutputStream stream(&output_string);
109  switch (proto_write_format) {
111  if (!proto.SerializeToZeroCopyStream(&stream)) {
112  LOG(WARNING) << "Serialize to stream failed.";
113  return false;
114  }
115  file_type_suffix = ".bin";
116  break;
118  if (!google::protobuf::TextFormat::PrintToString(proto, &output_string)) {
119  LOG(WARNING) << "Printing to string failed.";
120  return false;
121  }
122  break;
124  google::protobuf::util::JsonPrintOptions options;
125  options.add_whitespace = true;
126  options.always_print_primitive_fields = true;
127  options.preserve_proto_field_names = true;
128  if (!google::protobuf::util::MessageToJsonString(proto, &output_string,
129  options)
130  .ok()) {
131  LOG(WARNING) << "Printing to stream failed.";
132  return false;
133  }
134  file_type_suffix = ".json";
135  break;
136  }
137  if (gzipped) {
138  std::string gzip_string;
139  GzipString(output_string, &gzip_string);
140  output_string.swap(gzip_string);
141  file_type_suffix += ".gz";
142  }
143  std::string output_filename(filename);
144  if (append_extension_to_file_name) output_filename += file_type_suffix;
145  VLOG(1) << "Writing " << output_string.size() << " bytes to "
146  << output_filename;
147  if (!file::SetContents(output_filename, output_string, file::Defaults())
148  .ok()) {
149  LOG(WARNING) << "Writing to file failed.";
150  return false;
151  }
152  return true;
153 }
154 
155 } // namespace operations_research
#define CHECK_OK(x)
Definition: base/logging.h:40
#define LOG(severity)
Definition: base/logging.h:420
#define VLOG(verboselevel)
Definition: base/logging.h:978
CpModelProto proto
void GzipString(absl::string_view uncompressed, std::string *compressed)
Definition: gzipstring.h:63
bool GunzipString(const std::string &str, std::string *out)
Definition: gzipstring.h:22
const int WARNING
Definition: log_severity.h:31
absl::Status GetContents(const absl::string_view &filename, std::string *output, int flags)
Definition: file.cc:163
int Defaults()
Definition: base/file.h:119
absl::Status SetContents(const absl::string_view &filename, const absl::string_view &contents, int flags)
Definition: file.cc:196
The vehicle routing library lets one model and solve generic vehicle routing problems ranging from th...
absl::StatusOr< std::string > ReadFileToString(absl::string_view filename)
Definition: file_util.cc:32
bool WriteProtoToFile(absl::string_view filename, const google::protobuf::Message &proto, ProtoWriteFormat proto_write_format, bool gzipped, bool append_extension_to_file_name)
Definition: file_util.cc:102
bool ReadFileToProto(absl::string_view filename, google::protobuf::Message *proto)
Definition: file_util.cc:43
#define RETURN_IF_ERROR(expr)
Definition: status_macros.h:27