module-loader.c++ 11.9 KB
Newer Older
Kenton Varda's avatar
Kenton Varda committed
1 2
// Copyright (c) 2013-2014 Sandstorm Development Group, Inc. and contributors
// Licensed under the MIT License:
3
//
Kenton Varda's avatar
Kenton Varda committed
4 5 6 7 8 9
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
10
//
Kenton Varda's avatar
Kenton Varda committed
11 12
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
13
//
Kenton Varda's avatar
Kenton Varda committed
14 15 16 17 18 19 20
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.
21 22 23 24 25 26 27 28 29 30

#include "module-loader.h"
#include "lexer.h"
#include "parser.h"
#include <kj/vector.h>
#include <kj/mutex.h>
#include <kj/debug.h>
#include <kj/io.h>
#include <capnp/message.h>
#include <map>
31
#include <kj/miniposix.h>
32 33 34 35 36
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <errno.h>

37 38 39 40 41 42
#if _WIN32
#include <windows.h>
#else
#include <sys/mman.h>
#endif

43 44 45 46 47 48 49 50 51
namespace capnp {
namespace compiler {

namespace {

class MmapDisposer: public kj::ArrayDisposer {
protected:
  void disposeImpl(void* firstElement, size_t elementSize, size_t elementCount,
                   size_t capacity, void (*destroyElement)(void*)) const {
52 53 54
#if _WIN32
    KJ_ASSERT(UnmapViewOfFile(firstElement));
#else
55
    munmap(firstElement, elementSize * elementCount);
56
#endif
57 58 59
  }
};

60
KJ_CONSTEXPR(static const) MmapDisposer mmapDisposer = MmapDisposer();
61

62
kj::Array<const byte> mmapForRead(kj::StringPtr filename) {
63 64 65 66 67 68 69 70
  int fd;
  // We already established that the file exists, so this should not fail.
  KJ_SYSCALL(fd = open(filename.cStr(), O_RDONLY), filename);
  kj::AutoCloseFd closer(fd);

  struct stat stats;
  KJ_SYSCALL(fstat(fd, &stats));

71
  if (S_ISREG(stats.st_mode)) {
Kenton Varda's avatar
Kenton Varda committed
72 73 74 75 76
    if (stats.st_size == 0) {
      // mmap()ing zero bytes will fail.
      return nullptr;
    }

77
    // Regular file.  Just mmap() it.
78 79 80 81 82 83 84 85 86
#if _WIN32
    HANDLE handle = reinterpret_cast<HANDLE>(_get_osfhandle(fd));
    KJ_ASSERT(handle != INVALID_HANDLE_VALUE);
    HANDLE mappingHandle = CreateFileMapping(
        handle, NULL, PAGE_READONLY, 0, stats.st_size, NULL);
    KJ_ASSERT(mappingHandle != INVALID_HANDLE_VALUE);
    KJ_DEFER(KJ_ASSERT(CloseHandle(mappingHandle)));
    const void* mapping = MapViewOfFile(mappingHandle, FILE_MAP_READ, 0, 0, stats.st_size);
#else  // _WIN32
87 88 89 90
    const void* mapping = mmap(NULL, stats.st_size, PROT_READ, MAP_SHARED, fd, 0);
    if (mapping == MAP_FAILED) {
      KJ_FAIL_SYSCALL("mmap", errno, filename);
    }
91
#endif  // _WIN32, else
92

93 94
    return kj::Array<const byte>(
        reinterpret_cast<const byte*>(mapping), stats.st_size, mmapDisposer);
95 96 97
  } else {
    // This could be a stream of some sort, like a pipe.  Fall back to read().
    // TODO(cleanup):  This does a lot of copies.  Not sure I care.
98
    kj::Vector<byte> data(8192);
99

100
    byte buffer[4096];
101
    for (;;) {
102
      kj::miniposix::ssize_t n;
103 104 105 106 107 108 109
      KJ_SYSCALL(n = read(fd, buffer, sizeof(buffer)));
      if (n == 0) break;
      data.addAll(buffer, buffer + n);
    }

    return data.releaseAsArray();
  }
110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193
}

static char* canonicalizePath(char* path) {
  // Taken from some old C code of mine.

  // Preconditions:
  // - path has already been determined to be relative, perhaps because the pointer actually points
  //   into the middle of some larger path string, in which case it must point to the character
  //   immediately after a '/'.

  // Invariants:
  // - src points to the beginning of a path component.
  // - dst points to the location where the path component should end up, if it is not special.
  // - src == path or src[-1] == '/'.
  // - dst == path or dst[-1] == '/'.

  char* src = path;
  char* dst = path;
  char* locked = dst;  // dst cannot backtrack past this
  char* partEnd;
  bool hasMore;

  for (;;) {
    while (*src == '/') {
      // Skip duplicate slash.
      ++src;
    }

    partEnd = strchr(src, '/');
    hasMore = partEnd != NULL;
    if (hasMore) {
      *partEnd = '\0';
    } else {
      partEnd = src + strlen(src);
    }

    if (strcmp(src, ".") == 0) {
      // Skip it.
    } else if (strcmp(src, "..") == 0) {
      if (dst > locked) {
        // Backtrack over last path component.
        --dst;
        while (dst > locked && dst[-1] != '/') --dst;
      } else {
        locked += 3;
        goto copy;
      }
    } else {
      // Copy if needed.
    copy:
      if (dst < src) {
        memmove(dst, src, partEnd - src);
        dst += partEnd - src;
      } else {
        dst = partEnd;
      }
      *dst++ = '/';
    }

    if (hasMore) {
      src = partEnd + 1;
    } else {
      // Oops, we have to remove the trailing '/'.
      if (dst == path) {
        // Oops, there is no trailing '/'.  We have to return ".".
        strcpy(path, ".");
        return path + 1;
      } else {
        // Remove the trailing '/'.  Note that this means that opening the file will work even
        // if it is not a directory, where normally it should fail on non-directories when a
        // trailing '/' is present.  If this is a problem, we need to add some sort of special
        // handling for this case where we stat() it separately to check if it is a directory,
        // because Ekam findInput will not accept a trailing '/'.
        --dst;
        *dst = '\0';
        return dst;
      }
    }
  }
}

kj::String canonicalizePath(kj::StringPtr path) {
  KJ_STACK_ARRAY(char, result, path.size() + 1, 128, 512);
  strcpy(result.begin(), path.begin());
Kenton Varda's avatar
Kenton Varda committed
194 195 196

  char* start = path.startsWith("/") ? result.begin() + 1 : result.begin();
  char* end = canonicalizePath(start);
197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217
  return kj::heapString(result.slice(0, end - result.begin()));
}

kj::String catPath(kj::StringPtr base, kj::StringPtr add) {
  if (add.size() > 0 && add[0] == '/') {
    return kj::heapString(add);
  }

  const char* pos = base.end();
  while (pos > base.begin() && pos[-1] != '/') {
    --pos;
  }

  return kj::str(base.slice(0, pos - base.begin()), add);
}

}  // namespace


class ModuleLoader::Impl {
public:
218
  Impl(GlobalErrorReporter& errorReporter): errorReporter(errorReporter) {}
219 220 221 222 223

  void addImportPath(kj::String path) {
    searchPath.add(kj::heapString(kj::mv(path)));
  }

224 225
  kj::Maybe<Module&> loadModule(kj::StringPtr localName, kj::StringPtr sourceName);
  kj::Maybe<Module&> loadModuleFromSearchPath(kj::StringPtr sourceName);
226 227
  kj::Maybe<kj::Array<const byte>> readEmbed(kj::StringPtr localName, kj::StringPtr sourceName);
  kj::Maybe<kj::Array<const byte>> readEmbedFromSearchPath(kj::StringPtr sourceName);
228
  GlobalErrorReporter& getErrorReporter() { return errorReporter; }
229 230

private:
231
  GlobalErrorReporter& errorReporter;
232
  kj::Vector<kj::String> searchPath;
233
  std::map<kj::StringPtr, kj::Own<Module>> modules;
234 235
};

236
class ModuleLoader::ModuleImpl final: public Module {
237
public:
238
  ModuleImpl(ModuleLoader::Impl& loader, kj::String localName, kj::String sourceName)
239 240
      : loader(loader), localName(kj::mv(localName)), sourceName(kj::mv(sourceName)) {}

241
  kj::StringPtr getLocalName() {
242 243 244
    return localName;
  }

245
  kj::StringPtr getSourceName() override {
246 247 248
    return sourceName;
  }

249
  Orphan<ParsedFile> loadContent(Orphanage orphanage) override {
250
    kj::Array<const char> content = mmapForRead(localName).releaseAsChars();
251

252 253
    lineBreaks = nullptr;  // In case loadContent() is called multiple times.
    lineBreaks = lineBreaksSpace.construct(content);
254 255 256 257 258 259 260 261 262 263

    MallocMessageBuilder lexedBuilder;
    auto statements = lexedBuilder.initRoot<LexedStatements>();
    lex(content, statements, *this);

    auto parsed = orphanage.newOrphan<ParsedFile>();
    parseFile(statements.getStatements(), parsed.get(), *this);
    return parsed;
  }

264
  kj::Maybe<Module&> importRelative(kj::StringPtr importPath) override {
265 266 267 268 269 270 271
    if (importPath.size() > 0 && importPath[0] == '/') {
      return loader.loadModuleFromSearchPath(importPath.slice(1));
    } else {
      return loader.loadModule(catPath(localName, importPath), catPath(sourceName, importPath));
    }
  }

272 273 274 275 276 277 278 279
  kj::Maybe<kj::Array<const byte>> embedRelative(kj::StringPtr embedPath) override {
    if (embedPath.size() > 0 && embedPath[0] == '/') {
      return loader.readEmbedFromSearchPath(embedPath.slice(1));
    } else {
      return loader.readEmbed(catPath(localName, embedPath), catPath(sourceName, embedPath));
    }
  }

280 281 282
  void addError(uint32_t startByte, uint32_t endByte, kj::StringPtr message) override {
    auto& lines = *KJ_REQUIRE_NONNULL(lineBreaks,
        "Can't report errors until loadContent() is called.");
283

284
    loader.getErrorReporter().addError(
285
        localName, lines.toSourcePos(startByte), lines.toSourcePos(endByte), message);
286 287
  }

288
  bool hadErrors() override {
289
    return loader.getErrorReporter().hadErrors();
290 291 292
  }

private:
293
  ModuleLoader::Impl& loader;
294 295 296
  kj::String localName;
  kj::String sourceName;

297 298
  kj::SpaceFor<LineBreakTable> lineBreaksSpace;
  kj::Maybe<kj::Own<LineBreakTable>> lineBreaks;
299 300 301 302
};

// =======================================================================================

303 304
kj::Maybe<Module&> ModuleLoader::Impl::loadModule(
    kj::StringPtr localName, kj::StringPtr sourceName) {
305 306 307
  kj::String canonicalLocalName = canonicalizePath(localName);
  kj::String canonicalSourceName = canonicalizePath(sourceName);

308 309
  auto iter = modules.find(canonicalLocalName);
  if (iter != modules.end()) {
310 311 312 313
    // Return existing file.
    return *iter->second;
  }

314
  if (access(canonicalLocalName.cStr(), F_OK) < 0) {
315 316 317 318 319 320 321
    // No such file.
    return nullptr;
  }

  auto module = kj::heap<ModuleImpl>(
      *this, kj::mv(canonicalLocalName), kj::mv(canonicalSourceName));
  auto& result = *module;
322
  modules.insert(std::make_pair(result.getLocalName(), kj::mv(module)));
323 324 325
  return result;
}

326
kj::Maybe<Module&> ModuleLoader::Impl::loadModuleFromSearchPath(kj::StringPtr sourceName) {
327 328 329 330 331 332 333 334 335 336 337 338
  for (auto& search: searchPath) {
    kj::String candidate = kj::str(search, "/", sourceName);
    char* end = canonicalizePath(candidate.begin() + (candidate[0] == '/'));

    KJ_IF_MAYBE(module, loadModule(
        kj::heapString(candidate.slice(0, end - candidate.begin())), sourceName)) {
      return *module;
    }
  }
  return nullptr;
}

339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365
kj::Maybe<kj::Array<const byte>> ModuleLoader::Impl::readEmbed(
    kj::StringPtr localName, kj::StringPtr sourceName) {
  kj::String canonicalLocalName = canonicalizePath(localName);
  kj::String canonicalSourceName = canonicalizePath(sourceName);

  if (access(canonicalLocalName.cStr(), F_OK) < 0) {
    // No such file.
    return nullptr;
  }

  return mmapForRead(localName);
}

kj::Maybe<kj::Array<const byte>> ModuleLoader::Impl::readEmbedFromSearchPath(
    kj::StringPtr sourceName) {
  for (auto& search: searchPath) {
    kj::String candidate = kj::str(search, "/", sourceName);
    char* end = canonicalizePath(candidate.begin() + (candidate[0] == '/'));

    KJ_IF_MAYBE(module, readEmbed(
        kj::heapString(candidate.slice(0, end - candidate.begin())), sourceName)) {
      return kj::mv(*module);
    }
  }
  return nullptr;
}

366 367
// =======================================================================================

368
ModuleLoader::ModuleLoader(GlobalErrorReporter& errorReporter)
369
    : impl(kj::heap<Impl>(errorReporter)) {}
370
ModuleLoader::~ModuleLoader() noexcept(false) {}
371 372 373

void ModuleLoader::addImportPath(kj::String path) { impl->addImportPath(kj::mv(path)); }

374
kj::Maybe<Module&> ModuleLoader::loadModule(kj::StringPtr localName, kj::StringPtr sourceName) {
375 376 377 378 379
  return impl->loadModule(localName, sourceName);
}

}  // namespace compiler
}  // namespace capnp