1 : #include "references.hh"
2 : #include "pathlocks.hh"
3 : #include "misc.hh"
4 : #include "globals.hh"
5 : #include "local-store.hh"
6 : #include "util.hh"
7 :
8 : #include <map>
9 : #include <iostream>
10 : #include <sstream>
11 : #include <boost/shared_ptr.hpp>
12 : #include <boost/weak_ptr.hpp>
13 : #include <boost/enable_shared_from_this.hpp>
14 :
15 : #include <time.h>
16 : #include <sys/time.h>
17 : #include <sys/types.h>
18 : #include <sys/stat.h>
19 : #include <fcntl.h>
20 : #include <unistd.h>
21 : #include <errno.h>
22 :
23 : #include <pwd.h>
24 : #include <grp.h>
25 :
26 :
27 : /* Includes required for chroot support. */
28 : #include "config.h"
29 :
30 : #if HAVE_SYS_PARAM_H
31 : #include <sys/param.h>
32 : #endif
33 : #if HAVE_SYS_MOUNT_H
34 : #include <sys/mount.h>
35 : #endif
36 :
37 : #define CHROOT_ENABLED HAVE_CHROOT && HAVE_SYS_MOUNT_H && defined(MS_BIND)
38 :
39 :
40 : namespace nix {
41 :
42 : using std::map;
43 :
44 :
45 : /* !!! TODO derivationFromPath shouldn't be used here */
46 :
47 :
48 1704 : static string pathNullDevice = "/dev/null";
49 :
50 :
51 : static const uid_t rootUserId = 0;
52 :
53 :
54 : /* Forward definition. */
55 : class Worker;
56 :
57 :
58 : /* A pointer to a goal. */
59 : class Goal;
60 : typedef boost::shared_ptr<Goal> GoalPtr;
61 : typedef boost::weak_ptr<Goal> WeakGoalPtr;
62 :
63 : /* Set of goals. */
64 : typedef set<GoalPtr> Goals;
65 : typedef set<WeakGoalPtr> WeakGoals;
66 :
67 : /* A map of paths to goals (and the other way around). */
68 : typedef map<Path, WeakGoalPtr> WeakGoalMap;
69 :
70 :
71 :
72 : class Goal : public boost::enable_shared_from_this<Goal>
73 : {
74 : public:
75 : typedef enum {ecBusy, ecSuccess, ecFailed} ExitCode;
76 :
77 : protected:
78 :
79 : /* Backlink to the worker. */
80 : Worker & worker;
81 :
82 : /* Goals that this goal is waiting for. */
83 : Goals waitees;
84 :
85 : /* Goals waiting for this one to finish. Must use weak pointers
86 : here to prevent cycles. */
87 : WeakGoals waiters;
88 :
89 : /* Number of goals we are/were waiting for that have failed. */
90 : unsigned int nrFailed;
91 :
92 : /* Name of this goal for debugging purposes. */
93 : string name;
94 :
95 : /* Whether the goal is finished. */
96 : ExitCode exitCode;
97 :
98 611 : Goal(Worker & worker) : worker(worker)
99 : {
100 611 : nrFailed = 0;
101 611 : exitCode = ecBusy;
102 611 : forceInputs = false;
103 611 : }
104 :
105 611 : virtual ~Goal()
106 611 : {
107 611 : trace("goal destroyed");
108 611 : }
109 :
110 : bool forceInputs;
111 :
112 : public:
113 : virtual void work() = 0;
114 :
115 : void addWaitee(GoalPtr waitee);
116 :
117 : virtual void waiteeDone(GoalPtr waitee, ExitCode result);
118 :
119 0 : virtual void handleChildOutput(int fd, const string & data)
120 : {
121 0 : abort();
122 : }
123 :
124 0 : virtual void handleEOF(int fd)
125 : {
126 0 : abort();
127 : }
128 :
129 : void trace(const format & f);
130 :
131 0 : string getName()
132 : {
133 0 : return name;
134 : }
135 :
136 334 : ExitCode getExitCode()
137 : {
138 334 : return exitCode;
139 : }
140 :
141 : /* Cancel the goal. It should wake up its waiters, get rid of any
142 : running child processes that are being monitored by the worker
143 : (important!), etc. */
144 : virtual void cancel() = 0;
145 :
146 68 : void setForceInputs(bool x)
147 : {
148 68 : forceInputs = x;
149 68 : }
150 :
151 : protected:
152 : void amDone(ExitCode result);
153 : };
154 :
155 :
156 : /* A mapping used to remember for each child process to what goal it
157 : belongs, and file descriptors for receiving log data and output
158 : path creation commands. */
159 : struct Child
160 1188 : {
161 : WeakGoalPtr goal;
162 : set<int> fds;
163 : bool inBuildSlot;
164 : time_t lastOutput; /* time we last got output on stdout/stderr */
165 : };
166 :
167 : typedef map<pid_t, Child> Children;
168 :
169 :
170 : /* The worker class. */
171 : class Worker
172 : {
173 : private:
174 :
175 : /* Note: the worker should only have strong pointers to the
176 : top-level goals. */
177 :
178 : /* The top-level goals of the worker. */
179 : Goals topGoals;
180 :
181 : /* Goals that are ready to do some work. */
182 : WeakGoals awake;
183 :
184 : /* Goals waiting for a build slot. */
185 : WeakGoals wantingToBuild;
186 :
187 : /* Child processes currently running. */
188 : Children children;
189 :
190 : /* Number of build slots occupied. Not all child processes
191 : (namely build hooks) count as occupied build slots. */
192 : unsigned int nrChildren;
193 :
194 : /* Maps used to prevent multiple instantiations of a goal for the
195 : same derivation / path. */
196 : WeakGoalMap derivationGoals;
197 : WeakGoalMap substitutionGoals;
198 :
199 : /* Goals waiting for busy paths to be unlocked. */
200 : WeakGoals waitingForAnyGoal;
201 :
202 : public:
203 :
204 : LocalStore & store;
205 :
206 : Worker(LocalStore & store);
207 : ~Worker();
208 :
209 : /* Make a goal (with caching). */
210 : GoalPtr makeDerivationGoal(const Path & drvPath);
211 : GoalPtr makeSubstitutionGoal(const Path & storePath);
212 :
213 : /* Remove a dead goal. */
214 : void removeGoal(GoalPtr goal);
215 :
216 : /* Wake up a goal (i.e., there is something for it to do). */
217 : void wakeUp(GoalPtr goal);
218 :
219 : /* Can we start another child process? */
220 : bool canBuildMore();
221 :
222 : /* Registers a running child process. `inBuildSlot' means that
223 : the process counts towards the jobs limit. */
224 : void childStarted(GoalPtr goal, pid_t pid,
225 : const set<int> & fds, bool inBuildSlot);
226 :
227 : /* Unregisters a running child process. `wakeSleepers' should be
228 : false if there is no sense in waking up goals that are sleeping
229 : because they can't run yet (e.g., there is no free build slot,
230 : or the hook would still say `postpone'). */
231 : void childTerminated(pid_t pid, bool wakeSleepers = true);
232 :
233 : /* Put `goal' to sleep until a build slot becomes available (which
234 : might be right away). */
235 : void waitForBuildSlot(GoalPtr goal);
236 :
237 : /* Put `goal' to sleep until a child process terminates, i.e., a
238 : call is made to childTerminate(..., true). */
239 : void waitForChildTermination(GoalPtr goal);
240 :
241 : /* Wait for any goal to finish. Pretty indiscriminate way to
242 : wait for some resource that some other goal is holding. */
243 : void waitForAnyGoal(GoalPtr goal);
244 :
245 : /* Loop until the specified top-level goals have finished. */
246 : void run(const Goals & topGoals);
247 :
248 : /* Wait for input to become available. */
249 : void waitForInput();
250 :
251 : };
252 :
253 :
254 6 : MakeError(SubstError, Error)
255 12 : MakeError(BuildError, Error)
256 :
257 :
258 : //////////////////////////////////////////////////////////////////////
259 :
260 :
261 483 : void Goal::addWaitee(GoalPtr waitee)
262 : {
263 483 : waitees.insert(waitee);
264 483 : waitee->waiters.insert(shared_from_this());
265 483 : }
266 :
267 :
268 483 : void Goal::waiteeDone(GoalPtr waitee, ExitCode result)
269 : {
270 483 : assert(waitees.find(waitee) != waitees.end());
271 483 : waitees.erase(waitee);
272 :
273 : trace(format("waitee `%1%' done; %2% left") %
274 483 : waitee->name % waitees.size());
275 :
276 485 : if (result == ecFailed) ++nrFailed;
277 :
278 483 : if (waitees.empty() || (result == ecFailed && !keepGoing)) {
279 :
280 : /* If we failed and keepGoing is not set, we remove all
281 : remaining waitees. */
282 370 : for (Goals::iterator i = waitees.begin(); i != waitees.end(); ++i) {
283 0 : GoalPtr goal = *i;
284 0 : WeakGoals waiters2;
285 0 : for (WeakGoals::iterator j = goal->waiters.begin();
286 : j != goal->waiters.end(); ++j)
287 0 : if (j->lock() != shared_from_this())
288 0 : waiters2.insert(*j);
289 0 : goal->waiters = waiters2;
290 : }
291 370 : waitees.clear();
292 :
293 370 : worker.wakeUp(shared_from_this());
294 : }
295 483 : }
296 :
297 :
298 610 : void Goal::amDone(ExitCode result)
299 : {
300 610 : trace("done");
301 610 : assert(exitCode == ecBusy);
302 610 : assert(result == ecSuccess || result == ecFailed);
303 610 : exitCode = result;
304 1093 : for (WeakGoals::iterator i = waiters.begin(); i != waiters.end(); ++i) {
305 483 : GoalPtr goal = i->lock();
306 483 : if (goal) goal->waiteeDone(shared_from_this(), result);
307 : }
308 610 : waiters.clear();
309 610 : worker.removeGoal(shared_from_this());
310 610 : }
311 :
312 :
313 5146 : void Goal::trace(const format & f)
314 : {
315 5146 : debug(format("%1%: %2%") % name % f);
316 5146 : }
317 :
318 :
319 :
320 : //////////////////////////////////////////////////////////////////////
321 :
322 :
323 : /* Common initialisation performed in child processes. */
324 3 : void commonChildInit(Pipe & logPipe)
325 : {
326 : /* Put the child in a separate session (and thus a separate
327 : process group) so that it has no controlling terminal (meaning
328 : that e.g. ssh cannot open /dev/tty) and it doesn't receive
329 : terminal signals. */
330 3 : if (setsid() == -1)
331 0 : throw SysError(format("creating a new session"));
332 :
333 : /* Dup the write side of the logger pipe into stderr. */
334 3 : if (dup2(logPipe.writeSide, STDERR_FILENO) == -1)
335 0 : throw SysError("cannot pipe standard error into log file");
336 3 : logPipe.readSide.close();
337 :
338 : /* Dup stderr to stdout. */
339 3 : if (dup2(STDERR_FILENO, STDOUT_FILENO) == -1)
340 0 : throw SysError("cannot dup stderr into stdout");
341 :
342 : /* Reroute stdin to /dev/null. */
343 3 : int fdDevNull = open(pathNullDevice.c_str(), O_RDWR);
344 3 : if (fdDevNull == -1)
345 0 : throw SysError(format("cannot open `%1%'") % pathNullDevice);
346 3 : if (dup2(fdDevNull, STDIN_FILENO) == -1)
347 0 : throw SysError("cannot dup null device into stdin");
348 3 : }
349 :
350 :
351 : /* Convert a string list to an array of char pointers. Careful: the
352 : string list should outlive the array. */
353 0 : const char * * strings2CharPtrs(const Strings & ss)
354 : {
355 0 : const char * * arr = new const char * [ss.size() + 1];
356 0 : const char * * p = arr;
357 0 : for (Strings::const_iterator i = ss.begin(); i != ss.end(); ++i)
358 0 : *p++ = i->c_str();
359 0 : *p = 0;
360 0 : return arr;
361 : }
362 :
363 :
364 : /* Restore default handling of SIGPIPE, otherwise some programs will
365 : randomly say "Broken pipe". */
366 0 : static void restoreSIGPIPE()
367 : {
368 : struct sigaction act, oact;
369 0 : act.sa_handler = SIG_DFL;
370 0 : act.sa_flags = 0;
371 0 : if (sigaction(SIGPIPE, &act, &oact)) throw SysError("resetting SIGPIPE");
372 0 : }
373 :
374 :
375 : //////////////////////////////////////////////////////////////////////
376 :
377 :
378 : class UserLock
379 : {
380 : private:
381 : /* POSIX locks suck. If we have a lock on a file, and we open and
382 : close that file again (without closing the original file
383 : descriptor), we lose the lock. So we have to be *very* careful
384 : not to open a lock file on which we are holding a lock. */
385 : static PathSet lockedPaths; /* !!! not thread-safe */
386 :
387 : Path fnUserLock;
388 : AutoCloseFD fdUserLock;
389 :
390 : string user;
391 : uid_t uid;
392 : gid_t gid;
393 :
394 : public:
395 : UserLock();
396 : ~UserLock();
397 :
398 : void acquire();
399 : void release();
400 :
401 : void kill();
402 :
403 0 : string getUser() { return user; }
404 0 : uid_t getUID() { return uid; }
405 0 : uid_t getGID() { return gid; }
406 :
407 339 : bool enabled() { return uid != 0; }
408 :
409 : };
410 :
411 :
412 1151 : PathSet UserLock::lockedPaths;
413 :
414 :
415 220 : UserLock::UserLock()
416 : {
417 220 : uid = gid = 0;
418 220 : }
419 :
420 :
421 220 : UserLock::~UserLock()
422 : {
423 220 : release();
424 220 : }
425 :
426 :
427 0 : void UserLock::acquire()
428 : {
429 0 : assert(uid == 0);
430 :
431 0 : string buildUsersGroup = querySetting("build-users-group", "");
432 0 : assert(buildUsersGroup != "");
433 :
434 : /* Get the members of the build-users-group. */
435 0 : struct group * gr = getgrnam(buildUsersGroup.c_str());
436 0 : if (!gr)
437 : throw Error(format("the group `%1%' specified in `build-users-group' does not exist")
438 0 : % buildUsersGroup);
439 0 : gid = gr->gr_gid;
440 :
441 : /* Copy the result of getgrnam. */
442 0 : Strings users;
443 0 : for (char * * p = gr->gr_mem; *p; ++p) {
444 0 : debug(format("found build user `%1%'") % *p);
445 0 : users.push_back(*p);
446 : }
447 :
448 0 : if (users.empty())
449 : throw Error(format("the build users group `%1%' has no members")
450 0 : % buildUsersGroup);
451 :
452 : /* Find a user account that isn't currently in use for another
453 : build. */
454 0 : for (Strings::iterator i = users.begin(); i != users.end(); ++i) {
455 0 : debug(format("trying user `%1%'") % *i);
456 :
457 0 : struct passwd * pw = getpwnam(i->c_str());
458 0 : if (!pw)
459 : throw Error(format("the user `%1%' in the group `%2%' does not exist")
460 0 : % *i % buildUsersGroup);
461 :
462 0 : fnUserLock = (format("%1%/userpool/%2%") % nixStateDir % pw->pw_uid).str();
463 :
464 0 : if (lockedPaths.find(fnUserLock) != lockedPaths.end())
465 : /* We already have a lock on this one. */
466 0 : continue;
467 :
468 0 : AutoCloseFD fd = open(fnUserLock.c_str(), O_RDWR | O_CREAT, 0600);
469 0 : if (fd == -1)
470 0 : throw SysError(format("opening user lock `%1%'") % fnUserLock);
471 :
472 0 : if (lockFile(fd, ltWrite, false)) {
473 0 : fdUserLock = fd.borrow();
474 0 : lockedPaths.insert(fnUserLock);
475 0 : user = *i;
476 0 : uid = pw->pw_uid;
477 :
478 : /* Sanity check... */
479 0 : if (uid == getuid() || uid == geteuid())
480 : throw Error(format("the Nix user should not be a member of `%1%'")
481 0 : % buildUsersGroup);
482 :
483 : return;
484 : }
485 : }
486 :
487 : throw BuildError(format("all build users are currently in use; "
488 : "consider creating additional users and adding them to the `%1%' group")
489 0 : % buildUsersGroup);
490 : }
491 :
492 :
493 327 : void UserLock::release()
494 : {
495 327 : if (uid == 0) return;
496 0 : fdUserLock.close(); /* releases lock */
497 0 : assert(lockedPaths.find(fnUserLock) != lockedPaths.end());
498 0 : lockedPaths.erase(fnUserLock);
499 0 : fnUserLock = "";
500 0 : uid = 0;
501 : }
502 :
503 :
504 : static void runSetuidHelper(const string & command,
505 0 : const string & arg)
506 : {
507 : Path program = getEnv("NIX_SETUID_HELPER",
508 0 : nixLibexecDir + "/nix-setuid-helper");
509 :
510 : /* Fork. */
511 0 : Pid pid;
512 0 : pid = fork();
513 0 : switch (pid) {
514 :
515 : case -1:
516 0 : throw SysError("unable to fork");
517 :
518 : case 0: /* child */
519 : try {
520 0 : std::vector<const char *> args; /* careful with c_str()! */
521 0 : args.push_back(program.c_str());
522 0 : args.push_back(command.c_str());
523 0 : args.push_back(arg.c_str());
524 0 : args.push_back(0);
525 :
526 0 : restoreSIGPIPE();
527 :
528 0 : execve(program.c_str(), (char * *) &args[0], 0);
529 0 : throw SysError(format("executing `%1%'") % program);
530 : }
531 0 : catch (std::exception & e) {
532 0 : std::cerr << "error: " << e.what() << std::endl;
533 : }
534 0 : quickExit(1);
535 : }
536 :
537 : /* Parent. */
538 :
539 : /* Wait for the child to finish. */
540 0 : int status = pid.wait(true);
541 0 : if (!statusOk(status))
542 : throw Error(format("program `%1%' %2%")
543 0 : % program % statusToString(status));
544 0 : }
545 :
546 :
547 0 : void UserLock::kill()
548 : {
549 0 : assert(enabled());
550 0 : if (amPrivileged())
551 0 : killUser(uid);
552 : else
553 0 : runSetuidHelper("kill", user);
554 0 : }
555 :
556 :
557 0 : bool amPrivileged()
558 : {
559 0 : return geteuid() == 0;
560 : }
561 :
562 :
563 112 : bool haveBuildUsers()
564 : {
565 112 : return querySetting("build-users-group", "") != "";
566 : }
567 :
568 :
569 0 : void getOwnership(const Path & path)
570 : {
571 0 : runSetuidHelper("get-ownership", path);
572 0 : }
573 :
574 :
575 : void deletePathWrapped(const Path & path,
576 2790 : unsigned long long & bytesFreed, unsigned long long & blocksFreed)
577 : {
578 : try {
579 : /* First try to delete it ourselves. */
580 2790 : deletePath(path, bytesFreed, blocksFreed);
581 0 : } catch (SysError & e) {
582 : /* If this failed due to a permission error, then try it with
583 : the setuid helper. */
584 0 : if (haveBuildUsers() && !amPrivileged()) {
585 0 : getOwnership(path);
586 0 : deletePath(path, bytesFreed, blocksFreed);
587 : } else
588 0 : throw;
589 : }
590 2790 : }
591 :
592 :
593 115 : void deletePathWrapped(const Path & path)
594 : {
595 : unsigned long long dummy1, dummy2;
596 115 : deletePathWrapped(path, dummy1, dummy2);
597 115 : }
598 :
599 :
600 : //////////////////////////////////////////////////////////////////////
601 :
602 :
603 : /* Helper RAII class for automatically unmounting bind-mounts in
604 : chroots. */
605 : struct BindMount
606 : {
607 : Path source, target;
608 : Paths created;
609 :
610 : BindMount()
611 : {
612 : }
613 :
614 0 : BindMount(const Path & source, const Path & target)
615 0 : {
616 0 : bind(source, target);
617 0 : }
618 :
619 0 : ~BindMount()
620 : {
621 : try {
622 0 : unbind();
623 0 : } catch (...) {
624 0 : ignoreException();
625 : }
626 0 : }
627 :
628 0 : void bind(const Path & source, const Path & target)
629 : {
630 : #if CHROOT_ENABLED
631 0 : debug(format("bind mounting `%1%' to `%2%'") % source % target);
632 :
633 0 : this->source = source;
634 0 : this->target = target;
635 :
636 0 : created = createDirs(target);
637 :
638 0 : if (mount(source.c_str(), target.c_str(), "", MS_BIND, 0) == -1)
639 0 : throw SysError(format("bind mount from `%1%' to `%2%' failed") % source % target);
640 : #endif
641 0 : }
642 :
643 0 : void unbind()
644 : {
645 : #if CHROOT_ENABLED
646 0 : if (source == "") return;
647 :
648 0 : debug(format("unmount bind-mount `%1%'") % target);
649 :
650 : /* Urgh. Unmount sometimes doesn't succeed right away because
651 : the mount point is still busy. It shouldn't be, because
652 : we've killed all the build processes by now (at least when
653 : using a build user; see the check in killUser()). But
654 : maybe this is because those processes are still zombies and
655 : are keeping some kernel structures busy (open files,
656 : current directories, etc.). So retry a few times
657 : (actually, a 1 second sleep is almost certainly enough for
658 : the zombies to be cleaned up). */
659 0 : unsigned int tries = 0;
660 0 : while (umount(target.c_str()) == -1) {
661 0 : if (errno == EBUSY && ++tries < 10) {
662 0 : printMsg(lvlError, format("unmounting `%1%' failed, retrying after 1 second...") % target);
663 0 : sleep(1);
664 : }
665 : else
666 0 : throw SysError(format("unmounting bind-mount `%1%' failed") % target);
667 : }
668 :
669 : /* Get rid of the directories for the mount point created in
670 : bind(). */
671 0 : for (Paths::reverse_iterator i = created.rbegin(); i != created.rend(); ++i) {
672 0 : debug(format("deleting `%1%'") % *i);
673 0 : if (remove(i->c_str()) == -1)
674 0 : throw SysError(format("cannot unlink `%1%'") % *i);
675 : }
676 :
677 0 : source = "";
678 : #endif
679 : }
680 : };
681 :
682 :
683 : //////////////////////////////////////////////////////////////////////
684 :
685 :
686 : class DerivationGoal : public Goal
687 : {
688 : private:
689 : /* The path of the derivation. */
690 : Path drvPath;
691 :
692 : /* The derivation stored at drvPath. */
693 : Derivation drv;
694 :
695 : /* The remainder is state held during the build. */
696 :
697 : /* Locks on the output paths. */
698 : PathLocks outputLocks;
699 :
700 : /* All input paths (that is, the union of FS closures of the
701 : immediate input paths). */
702 : PathSet inputPaths;
703 :
704 : /* Referenceable paths (i.e., input and output paths). */
705 : PathSet allPaths;
706 :
707 : /* User selected for running the builder. */
708 : UserLock buildUser;
709 :
710 : /* The process ID of the builder. */
711 : Pid pid;
712 :
713 : /* The temporary directory. */
714 : Path tmpDir;
715 :
716 : /* File descriptor for the log file. */
717 : AutoCloseFD fdLogFile;
718 :
719 : /* Pipe for the builder's standard output/error. */
720 : Pipe logPipe;
721 :
722 : /* Pipes for talking to the build hook (if any). */
723 : Pipe toHook;
724 : Pipe fromHook;
725 :
726 : /* Whether we're currently doing a chroot build. */
727 : bool useChroot;
728 :
729 : /* RAII objects to delete the chroot directory and its /tmp
730 : directory. Don't change the order: autoDelChrootTmp has to be
731 : deleted before autoDelChrootRoot. */
732 : boost::shared_ptr<AutoDelete> autoDelChrootRoot;
733 : boost::shared_ptr<AutoDelete> autoDelChrootTmp;
734 :
735 : /* In chroot builds, the list of bind mounts currently active.
736 : The destructor of BindMount will cause the binds to be
737 : unmounted. Keep this *below* autoDelChroot* just to be safe! */
738 : list<boost::shared_ptr<BindMount> > bindMounts;
739 :
740 : typedef void (DerivationGoal::*GoalState)();
741 : GoalState state;
742 :
743 : public:
744 : DerivationGoal(const Path & drvPath, Worker & worker);
745 : ~DerivationGoal();
746 :
747 : void cancel();
748 :
749 : void work();
750 :
751 6 : Path getDrvPath()
752 : {
753 6 : return drvPath;
754 : }
755 :
756 : private:
757 : /* The states. */
758 : void init();
759 : void haveDerivation();
760 : void outputsSubstituted();
761 : void inputsRealised();
762 : void tryToBuild();
763 : void buildDone();
764 :
765 : /* Is the build hook willing to perform the build? */
766 : typedef enum {rpAccept, rpDecline, rpPostpone, rpDone, rpRestart} HookReply;
767 : HookReply tryBuildHook();
768 :
769 : /* Synchronously wait for a build hook to finish. */
770 : void terminateBuildHook(bool kill = false);
771 :
772 : /* Acquires locks on the output paths and gathers information
773 : about the build (e.g., the input closures). During this
774 : process its possible that we find out that the build is
775 : unnecessary, in which case we return prDone. It's also
776 : possible that some other goal is already building/substituting
777 : the output paths, in which case we return prRestart (go back to
778 : the haveDerivation() state). Otherwise, prProceed is
779 : returned. */
780 : typedef enum {prProceed, prDone, prRestart} PrepareBuildReply;
781 : PrepareBuildReply prepareBuild();
782 :
783 : /* Start building a derivation. */
784 : void startBuilder();
785 :
786 : /* Must be called after the output paths have become valid (either
787 : due to a successful build or hook, or because they already
788 : were). */
789 : void computeClosure();
790 :
791 : /* Open a log file and a pipe to it. */
792 : Path openLogFile();
793 :
794 : /* Common initialisation to be performed in child processes (i.e.,
795 : both in builders and in build hooks). */
796 : void initChild();
797 :
798 : /* Delete the temporary directory, if we have one. */
799 : void deleteTmpDir(bool force);
800 :
801 : /* Callback used by the worker to write to the log. */
802 : void handleChildOutput(int fd, const string & data);
803 : void handleEOF(int fd);
804 :
805 : /* Return the set of (in)valid paths. */
806 : PathSet checkPathValidity(bool returnValid);
807 :
808 : /* Forcibly kill the child process, if any. */
809 : void killChild();
810 : };
811 :
812 :
813 220 : DerivationGoal::DerivationGoal(const Path & drvPath, Worker & worker)
814 220 : : Goal(worker)
815 : {
816 220 : this->drvPath = drvPath;
817 220 : state = &DerivationGoal::init;
818 220 : name = (format("building of `%1%'") % drvPath).str();
819 220 : trace("created");
820 220 : }
821 :
822 220 : DerivationGoal::~DerivationGoal()
823 : {
824 : /* Careful: we should never ever throw an exception from a
825 : destructor. */
826 : try {
827 220 : killChild();
828 220 : deleteTmpDir(false);
829 0 : } catch (...) {
830 0 : ignoreException();
831 : }
832 220 : }
833 :
834 220 : void DerivationGoal::killChild()
835 : {
836 220 : if (pid != -1) {
837 0 : worker.childTerminated(pid);
838 :
839 0 : if (buildUser.enabled()) {
840 : /* We can't use pid.kill(), since we may not have the
841 : appropriate privilege. I.e., if we're not root, then
842 : setuid helper should do it).
843 :
844 : Also, if we're using a build user, then there is a
845 : tricky race condition: if we kill the build user before
846 : the child has done its setuid() to the build user uid,
847 : then it won't be killed, and we'll potentially lock up
848 : in pid.wait(). So also send a conventional kill to the
849 : child. */
850 0 : ::kill(-pid, SIGKILL); /* ignore the result */
851 0 : buildUser.kill();
852 0 : pid.wait(true);
853 : } else
854 0 : pid.kill();
855 :
856 0 : assert(pid == -1);
857 : }
858 220 : }
859 :
860 :
861 0 : void DerivationGoal::cancel()
862 : {
863 0 : killChild();
864 0 : amDone(ecFailed);
865 0 : }
866 :
867 :
868 890 : void DerivationGoal::work()
869 : {
870 890 : (this->*state)();
871 889 : }
872 :
873 :
874 220 : void DerivationGoal::init()
875 : {
876 220 : trace("init");
877 :
878 : /* The first thing to do is to make sure that the derivation
879 : exists. If it doesn't, it may be created through a
880 : substitute. */
881 440 : addWaitee(worker.makeSubstitutionGoal(drvPath));
882 :
883 220 : state = &DerivationGoal::haveDerivation;
884 220 : }
885 :
886 :
887 221 : void DerivationGoal::haveDerivation()
888 : {
889 221 : trace("loading derivation");
890 :
891 221 : if (nrFailed != 0) {
892 0 : printMsg(lvlError,
893 : format("cannot build missing derivation `%1%'")
894 : % drvPath);
895 0 : amDone(ecFailed);
896 0 : return;
897 : }
898 :
899 221 : assert(worker.store.isValidPath(drvPath));
900 :
901 : /* Get the derivation. */
902 221 : drv = derivationFromPath(drvPath);
903 :
904 442 : for (DerivationOutputs::iterator i = drv.outputs.begin();
905 : i != drv.outputs.end(); ++i)
906 221 : worker.store.addTempRoot(i->second.path);
907 :
908 : /* Check what outputs paths are not already valid. */
909 221 : PathSet invalidOutputs = checkPathValidity(false);
910 :
911 : /* If they are all valid, then we're done. */
912 221 : if (invalidOutputs.size() == 0) {
913 78 : if(!forceInputs) {
914 78 : amDone(ecSuccess);
915 78 : return;
916 : }
917 : }
918 :
919 : /* If this is a fixed-output derivation, it is possible that some
920 : other goal is already building the output paths. (The case
921 : where some other process is building it is handled through
922 : normal locking mechanisms.) So if any output paths are already
923 : being built, put this goal to sleep. */
924 286 : for (PathSet::iterator i = invalidOutputs.begin();
925 : i != invalidOutputs.end(); ++i)
926 143 : if (pathIsLockedByMe(*i)) {
927 : /* Wait until any goal finishes (hopefully the one that is
928 : locking *i), then retry haveDerivation(). */
929 0 : worker.waitForAnyGoal(shared_from_this());
930 : return;
931 : }
932 :
933 : /* We are first going to try to create the invalid output paths
934 : through substitutes. If that doesn't work, we'll build
935 : them. */
936 286 : for (PathSet::iterator i = invalidOutputs.begin();
937 : i != invalidOutputs.end(); ++i)
938 : /* Don't bother creating a substitution goal if there are no
939 : substitutes. */
940 143 : if (worker.store.hasSubstitutes(*i))
941 5 : addWaitee(worker.makeSubstitutionGoal(*i));
942 :
943 143 : if (waitees.empty()) /* to prevent hang (no wake-up event) */
944 138 : outputsSubstituted();
945 : else
946 5 : state = &DerivationGoal::outputsSubstituted;
947 : }
948 :
949 :
950 143 : void DerivationGoal::outputsSubstituted()
951 : {
952 143 : trace("all outputs substituted (maybe)");
953 :
954 145 : if (nrFailed > 0 && !tryFallback)
955 1 : throw Error(format("some substitutes for the outputs of derivation `%1%' failed; try `--fallback'") % drvPath);
956 :
957 142 : nrFailed = 0;
958 :
959 142 : if (checkPathValidity(false).size() == 0) {
960 3 : if (! forceInputs){
961 3 : amDone(ecSuccess);
962 3 : return;
963 : }
964 : }
965 :
966 : /* Otherwise, at least one of the output paths could not be
967 : produced using a substitute. So we have to build instead. */
968 :
969 : /* The inputs must be built before we can build this goal. */
970 : /* !!! but if possible, only install the paths that we need */
971 207 : for (DerivationInputs::iterator i = drv.inputDrvs.begin();
972 : i != drv.inputDrvs.end(); ++i){
973 68 : GoalPtr newGoal = worker.makeDerivationGoal(i->first);
974 68 : newGoal->setForceInputs(forceInputs);
975 68 : addWaitee(newGoal);
976 : }
977 :
978 642 : for (PathSet::iterator i = drv.inputSrcs.begin();
979 : i != drv.inputSrcs.end(); ++i)
980 182 : addWaitee(worker.makeSubstitutionGoal(*i));
981 :
982 : /* Actually, I do some work twice just to be on the safe side */
983 139 : string s = drv.env["exportBuildReferencesGraph"];
984 278 : Strings ss = tokenizeString(s);
985 278 : if (ss.size() % 2 !=0)
986 0 : throw BuildError(format("odd number of tokens in `exportBuildReferencesGraph': `%1%'") % s);
987 139 : for (Strings::iterator i = ss.begin(); i != ss.end(); ) {
988 0 : string fileName = *i++;
989 0 : Path storePath=*i++;
990 :
991 0 : if (!isInStore(storePath))
992 : throw BuildError(format("`exportBuildReferencesGraph' contains a non-store path `%1%'")
993 0 : % storePath);
994 0 : storePath = toStorePath(storePath);
995 0 : if (!worker.store.isValidPath(storePath))
996 : throw BuildError(format("`exportBuildReferencesGraph' contains an invalid path `%1%'")
997 0 : % storePath);
998 :
999 : /* Build-time closure should be in dependencies
1000 : * We really want just derivation, its closure
1001 : * and outputs. Looks like we should build it.
1002 : * */
1003 :
1004 0 : GoalPtr newGoal = worker.makeDerivationGoal(storePath);
1005 0 : newGoal->setForceInputs(true);
1006 0 : addWaitee(newGoal);
1007 : }
1008 :
1009 139 : state = &DerivationGoal::inputsRealised;
1010 : }
1011 :
1012 :
1013 139 : void DerivationGoal::inputsRealised()
1014 : {
1015 139 : trace("all inputs realised");
1016 :
1017 139 : if (nrFailed != 0) {
1018 0 : printMsg(lvlError,
1019 : format("cannot build derivation `%1%': "
1020 : "%2% inputs could not be realised")
1021 : % drvPath % nrFailed);
1022 0 : amDone(ecFailed);
1023 0 : return;
1024 : }
1025 :
1026 : /* Maybe we just wanted to force build of inputs */
1027 139 : if (checkPathValidity(false).size() == 0) {
1028 0 : amDone(ecSuccess);
1029 0 : return;
1030 : }
1031 :
1032 : /* Okay, try to build. Note that here we don't wait for a build
1033 : slot to become available, since we don't need one if there is a
1034 : build hook. */
1035 139 : state = &DerivationGoal::tryToBuild;
1036 139 : worker.wakeUp(shared_from_this());
1037 : }
1038 :
1039 :
1040 192 : void DerivationGoal::tryToBuild()
1041 : {
1042 192 : trace("trying to build");
1043 :
1044 : try {
1045 :
1046 : /* Is the build hook willing to accept this job? */
1047 384 : switch (tryBuildHook()) {
1048 : case rpAccept:
1049 : /* Yes, it has started doing so. Wait until we get
1050 : EOF from the hook. */
1051 1 : state = &DerivationGoal::buildDone;
1052 1 : return;
1053 : case rpPostpone:
1054 : /* Not now; wait until at least one child finishes. */
1055 0 : worker.waitForChildTermination(shared_from_this());
1056 0 : return;
1057 : case rpDecline:
1058 : /* We should do it ourselves. */
1059 191 : break;
1060 : case rpDone:
1061 : /* Somebody else did it. */
1062 0 : amDone(ecSuccess);
1063 0 : return;
1064 : case rpRestart:
1065 : /* Somebody else is building this output path.
1066 : Restart from haveDerivation(). */
1067 0 : state = &DerivationGoal::haveDerivation;
1068 0 : worker.waitForAnyGoal(shared_from_this());
1069 0 : return;
1070 : }
1071 :
1072 : /* Make sure that we are allowed to start a build. */
1073 191 : if (!worker.canBuildMore()) {
1074 53 : worker.waitForBuildSlot(shared_from_this());
1075 53 : return;
1076 : }
1077 :
1078 : /* Acquire locks and such. If we then see that the build has
1079 : been done by somebody else, we're done. */
1080 138 : PrepareBuildReply preply = prepareBuild();
1081 138 : if (preply == prDone) {
1082 25 : amDone(ecSuccess);
1083 25 : return;
1084 113 : } else if (preply == prRestart) {
1085 1 : state = &DerivationGoal::haveDerivation;
1086 1 : worker.waitForAnyGoal(shared_from_this());
1087 1 : return;
1088 : }
1089 :
1090 : /* Okay, we have to build. */
1091 112 : startBuilder();
1092 :
1093 0 : } catch (BuildError & e) {
1094 0 : printMsg(lvlError, e.msg());
1095 0 : if (printBuildTrace) {
1096 0 : printMsg(lvlError, format("@ build-failed %1% %2% %3% %4%")
1097 : % drvPath % drv.outputs["out"].path % 0 % e.msg());
1098 : }
1099 0 : amDone(ecFailed);
1100 0 : return;
1101 : }
1102 :
1103 : /* This state will be reached when we get EOF on the child's
1104 : log pipe. */
1105 112 : state = &DerivationGoal::buildDone;
1106 : }
1107 :
1108 :
1109 113 : void DerivationGoal::buildDone()
1110 : {
1111 113 : trace("build done");
1112 :
1113 : /* Since we got an EOF on the logger pipe, the builder is presumed
1114 : to have terminated. In fact, the builder could also have
1115 : simply have closed its end of the pipe --- just don't do that
1116 : :-) */
1117 : /* !!! this could block! security problem! solution: kill the
1118 : child */
1119 113 : pid_t savedPid = pid;
1120 113 : int status = pid.wait(true);
1121 :
1122 113 : debug(format("builder process for `%1%' finished") % drvPath);
1123 :
1124 : /* So the child is gone now. */
1125 113 : worker.childTerminated(savedPid);
1126 :
1127 : /* Close the read side of the logger pipe. */
1128 113 : logPipe.readSide.close();
1129 :
1130 : /* Close the log file. */
1131 113 : fdLogFile.close();
1132 :
1133 : /* When running under a build user, make sure that all processes
1134 : running under that uid are gone. This is to prevent a
1135 : malicious user from leaving behind a process that keeps files
1136 : open and modifies them after they have been chown'ed to
1137 : root. */
1138 113 : if (buildUser.enabled())
1139 0 : buildUser.kill();
1140 :
1141 : try {
1142 :
1143 : /* Some cleanup per path. We do this here and not in
1144 : computeClosure() for convenience when the build has
1145 : failed. */
1146 226 : for (DerivationOutputs::iterator i = drv.outputs.begin();
1147 : i != drv.outputs.end(); ++i)
1148 : {
1149 113 : Path path = i->second.path;
1150 113 : if (!pathExists(path)) continue;
1151 :
1152 : struct stat st;
1153 113 : if (lstat(path.c_str(), &st) == -1)
1154 0 : throw SysError(format("getting attributes of path `%1%'") % path);
1155 :
1156 : #ifndef __CYGWIN__
1157 : /* Check that the output is not group or world writable,
1158 : as that means that someone else can have interfered
1159 : with the build. Also, the output should be owned by
1160 : the build user. */
1161 113 : if ((!S_ISLNK(st.st_mode) && (st.st_mode & (S_IWGRP | S_IWOTH))) ||
1162 : (buildUser.enabled() && st.st_uid != buildUser.getUID()))
1163 0 : throw BuildError(format("suspicious ownership or permission on `%1%'; rejecting this build output") % path);
1164 : #endif
1165 :
1166 : /* Gain ownership of the build result using the setuid
1167 : wrapper if we're not root. If we *are* root, then
1168 : canonicalisePathMetaData() will take care of this later
1169 : on. */
1170 113 : if (buildUser.enabled() && !amPrivileged())
1171 0 : getOwnership(path);
1172 : }
1173 :
1174 : /* Check the exit status. */
1175 113 : if (!statusOk(status)) {
1176 3 : deleteTmpDir(false);
1177 : throw BuildError(format("builder for `%1%' %2%")
1178 3 : % drvPath % statusToString(status));
1179 : }
1180 :
1181 110 : deleteTmpDir(true);
1182 :
1183 : /* In chroot builds, unmount the bind mounts ASAP. */
1184 110 : bindMounts.clear(); /* the destructors will do the rest */
1185 :
1186 : /* Compute the FS closure of the outputs and register them as
1187 : being valid. */
1188 110 : computeClosure();
1189 :
1190 12 : } catch (BuildError & e) {
1191 6 : printMsg(lvlError, e.msg());
1192 6 : if (printBuildTrace) {
1193 0 : printMsg(lvlError, format("@ build-failed %1% %2% %3% %4%")
1194 : % drvPath % drv.outputs["out"].path % status % e.msg());
1195 : }
1196 6 : amDone(ecFailed);
1197 6 : return;
1198 : }
1199 :
1200 : /* Release the build user, if applicable. */
1201 107 : buildUser.release();
1202 :
1203 107 : if (printBuildTrace) {
1204 0 : printMsg(lvlError, format("@ build-succeeded %1% %2%")
1205 : % drvPath % drv.outputs["out"].path);
1206 : }
1207 :
1208 107 : amDone(ecSuccess);
1209 : }
1210 :
1211 :
1212 3 : static string readLine(int fd)
1213 : {
1214 3 : string s;
1215 20 : while (1) {
1216 23 : checkInterrupt();
1217 : char ch;
1218 23 : ssize_t rd = read(fd, &ch, 1);
1219 23 : if (rd == -1) {
1220 0 : if (errno != EINTR)
1221 0 : throw SysError("reading a line");
1222 23 : } else if (rd == 0)
1223 0 : throw Error("unexpected EOF reading a line");
1224 : else {
1225 23 : if (ch == '\n') return s;
1226 20 : s += ch;
1227 : }
1228 0 : }
1229 : }
1230 :
1231 :
1232 1 : static void writeLine(int fd, string s)
1233 : {
1234 1 : s += '\n';
1235 1 : writeFull(fd, (const unsigned char *) s.c_str(), s.size());
1236 1 : }
1237 :
1238 :
1239 : /* !!! ugly hack */
1240 4 : static void drain(int fd)
1241 : {
1242 : unsigned char buffer[1024];
1243 2 : while (1) {
1244 4 : checkInterrupt();
1245 4 : ssize_t rd = read(fd, buffer, sizeof buffer);
1246 4 : if (rd == -1) {
1247 0 : if (errno != EINTR)
1248 0 : throw SysError("draining");
1249 4 : } else if (rd == 0) break;
1250 2 : else writeToStderr(buffer, rd);
1251 : }
1252 2 : }
1253 :
1254 :
1255 389 : PathSet outputPaths(const DerivationOutputs & outputs)
1256 : {
1257 389 : PathSet paths;
1258 778 : for (DerivationOutputs::const_iterator i = outputs.begin();
1259 : i != outputs.end(); ++i)
1260 389 : paths.insert(i->second.path);
1261 0 : return paths;
1262 : }
1263 :
1264 :
1265 192 : DerivationGoal::HookReply DerivationGoal::tryBuildHook()
1266 : {
1267 192 : if (!useBuildHook) return rpDecline;
1268 147 : Path buildHook = getEnv("NIX_BUILD_HOOK");
1269 441 : if (buildHook == "") return rpDecline;
1270 3 : buildHook = absPath(buildHook);
1271 :
1272 : /* Create a directory where we will store files used for
1273 : communication between us and the build hook. */
1274 6 : tmpDir = createTempDir();
1275 :
1276 : /* Create the log file and pipe. */
1277 3 : Path logFile = openLogFile();
1278 :
1279 : /* Create the communication pipes. */
1280 3 : toHook.create();
1281 3 : fromHook.create();
1282 :
1283 : /* Fork the hook. */
1284 3 : pid = fork();
1285 6 : switch (pid) {
1286 :
1287 : case -1:
1288 0 : throw SysError("unable to fork");
1289 :
1290 : case 0:
1291 : try { /* child */
1292 :
1293 3 : initChild();
1294 :
1295 : execl(buildHook.c_str(), buildHook.c_str(),
1296 : (worker.canBuildMore() ? (string) "1" : "0").c_str(),
1297 : thisSystem.c_str(),
1298 : drv.platform.c_str(),
1299 3 : drvPath.c_str(), NULL);
1300 :
1301 0 : throw SysError(format("executing `%1%'") % buildHook);
1302 :
1303 0 : } catch (std::exception & e) {
1304 0 : std::cerr << format("build hook error: %1%") % e.what() << std::endl;
1305 : }
1306 0 : quickExit(1);
1307 : }
1308 :
1309 : /* parent */
1310 3 : pid.setSeparatePG(true);
1311 3 : pid.setKillSignal(SIGTERM);
1312 3 : logPipe.writeSide.close();
1313 : worker.childStarted(shared_from_this(),
1314 3 : pid, singleton<set<int> >(logPipe.readSide), false);
1315 :
1316 3 : fromHook.writeSide.close();
1317 3 : toHook.readSide.close();
1318 :
1319 : /* Read the first line of input, which should be a word indicating
1320 : whether the hook wishes to perform the build. !!! potential
1321 : for deadlock here: we should also read from the child's logger
1322 : pipe. */
1323 3 : string reply;
1324 : try {
1325 3 : reply = readLine(fromHook.readSide);
1326 0 : } catch (Error & e) {
1327 0 : terminateBuildHook(true);
1328 0 : throw;
1329 : }
1330 :
1331 3 : debug(format("hook reply is `%1%'") % reply);
1332 :
1333 3 : if (reply == "decline" || reply == "postpone") {
1334 : /* Clean up the child. !!! hacky / should verify */
1335 2 : terminateBuildHook();
1336 2 : return reply == "decline" ? rpDecline : rpPostpone;
1337 : }
1338 :
1339 1 : else if (reply == "accept") {
1340 :
1341 : /* Acquire locks and such. If we then see that the output
1342 : paths are now valid, we're done. */
1343 1 : PrepareBuildReply preply = prepareBuild();
1344 1 : if (preply == prDone || preply == prRestart) {
1345 : /* Tell the hook to exit. */
1346 0 : writeLine(toHook.writeSide, "cancel");
1347 0 : terminateBuildHook();
1348 0 : return preply == prDone ? rpDone : rpRestart;
1349 : }
1350 :
1351 1 : printMsg(lvlInfo, format("running hook to build path(s) %1%")
1352 : % showPaths(outputPaths(drv.outputs)));
1353 :
1354 : /* Write the information that the hook needs to perform the
1355 : build, i.e., the set of input paths, the set of output
1356 : paths, and the references (pointer graph) in the input
1357 : paths. */
1358 :
1359 1 : Path inputListFN = tmpDir + "/inputs";
1360 1 : Path outputListFN = tmpDir + "/outputs";
1361 1 : Path referencesFN = tmpDir + "/references";
1362 :
1363 : /* The `inputs' file lists all inputs that have to be copied
1364 : to the remote system. This unfortunately has to contain
1365 : the entire derivation closure to ensure that the validity
1366 : invariant holds on the remote system. (I.e., it's
1367 : unfortunate that we have to list it since the remote system
1368 : *probably* already has it.) */
1369 1 : PathSet allInputs;
1370 1 : allInputs.insert(inputPaths.begin(), inputPaths.end());
1371 1 : computeFSClosure(drvPath, allInputs);
1372 :
1373 1 : string s;
1374 3 : for (PathSet::iterator i = allInputs.begin();
1375 : i != allInputs.end(); ++i)
1376 2 : s += *i + "\n";
1377 :
1378 1 : writeStringToFile(inputListFN, s);
1379 :
1380 : /* The `outputs' file lists all outputs that have to be copied
1381 : from the remote system. */
1382 1 : s = "";
1383 2 : for (DerivationOutputs::iterator i = drv.outputs.begin();
1384 : i != drv.outputs.end(); ++i)
1385 1 : s += i->second.path + "\n";
1386 1 : writeStringToFile(outputListFN, s);
1387 :
1388 : /* The `references' file has exactly the format accepted by
1389 : `nix-store --register-validity'. */
1390 : writeStringToFile(referencesFN,
1391 1 : makeValidityRegistration(allInputs, true, false));
1392 :
1393 : /* Tell the hook to proceed. */
1394 1 : writeLine(toHook.writeSide, "okay");
1395 :
1396 1 : if (printBuildTrace) {
1397 0 : printMsg(lvlError, format("@ build-started %1% %2% %3% %4%")
1398 : % drvPath % drv.outputs["out"].path % drv.platform % logFile);
1399 : }
1400 :
1401 1 : return rpAccept;
1402 : }
1403 :
1404 0 : else throw Error(format("bad hook reply `%1%'") % reply);
1405 : }
1406 :
1407 :
1408 2 : void DerivationGoal::terminateBuildHook(bool kill)
1409 : {
1410 : /* !!! drain stdout of hook */
1411 2 : debug("terminating build hook");
1412 2 : pid_t savedPid = pid;
1413 2 : if (kill)
1414 0 : pid.kill();
1415 : else
1416 2 : pid.wait(true);
1417 : /* `false' means don't wake up waiting goals, since we want to
1418 : keep this build slot ourselves. */
1419 2 : worker.childTerminated(savedPid, false);
1420 2 : fromHook.readSide.close();
1421 2 : toHook.writeSide.close();
1422 2 : fdLogFile.close();
1423 2 : drain(logPipe.readSide);
1424 2 : logPipe.readSide.close();
1425 2 : deleteTmpDir(true); /* get rid of the hook's temporary directory */
1426 2 : }
1427 :
1428 :
1429 139 : DerivationGoal::PrepareBuildReply DerivationGoal::prepareBuild()
1430 : {
1431 : /* Check for the possibility that some other goal in this process
1432 : has locked the output since we checked in haveDerivation().
1433 : (It can't happen between here and the lockPaths() call below
1434 : because we're not allowing multi-threading.) */
1435 277 : for (DerivationOutputs::iterator i = drv.outputs.begin();
1436 : i != drv.outputs.end(); ++i)
1437 139 : if (pathIsLockedByMe(i->second.path)) {
1438 1 : debug(format("restarting derivation `%1%' because `%2%' is locked by another goal")
1439 : % drvPath % i->second.path);
1440 1 : return prRestart;
1441 : }
1442 :
1443 : /* Obtain locks on all output paths. The locks are automatically
1444 : released when we exit this function or Nix crashes. */
1445 : /* !!! BUG: this could block, which is not allowed. */
1446 : /* !!! and once we make this non-blocking, we should carefully
1447 : consider the case where some but not all locks are required; we
1448 : should then release the acquired locks so that the other
1449 : processes and the pathIsLockedByMe() test don't get confused. */
1450 : outputLocks.lockPaths(outputPaths(drv.outputs),
1451 138 : (format("waiting for lock on %1%") % showPaths(outputPaths(drv.outputs))).str());
1452 :
1453 : /* Now check again whether the outputs are valid. This is because
1454 : another process may have started building in parallel. After
1455 : it has finished and released the locks, we can (and should)
1456 : reuse its results. (Strictly speaking the first check can be
1457 : omitted, but that would be less efficient.) Note that since we
1458 : now hold the locks on the output paths, no other process can
1459 : build this derivation, so no further checks are necessary. */
1460 138 : PathSet validPaths = checkPathValidity(true);
1461 276 : if (validPaths.size() == drv.outputs.size()) {
1462 25 : debug(format("skipping build of derivation `%1%', someone beat us to it")
1463 : % drvPath);
1464 25 : outputLocks.setDeletion(true);
1465 163 : return prDone;
1466 : }
1467 :
1468 113 : if (validPaths.size() > 0) {
1469 : /* !!! fix this; try to delete valid paths */
1470 : throw BuildError(
1471 : format("derivation `%1%' is blocked by its output paths")
1472 0 : % drvPath);
1473 : }
1474 :
1475 : /* If any of the outputs already exist but are not registered,
1476 : delete them. */
1477 226 : for (DerivationOutputs::iterator i = drv.outputs.begin();
1478 : i != drv.outputs.end(); ++i)
1479 : {
1480 113 : Path path = i->second.path;
1481 113 : if (worker.store.isValidPath(path))
1482 0 : throw BuildError(format("obstructed build: path `%1%' exists") % path);
1483 113 : if (pathExists(path)) {
1484 0 : debug(format("removing unregistered path `%1%'") % path);
1485 0 : deletePathWrapped(path);
1486 : }
1487 : }
1488 :
1489 : /* Gather information necessary for computing the closure and/or
1490 : running the build hook. */
1491 :
1492 : /* The outputs are referenceable paths. */
1493 226 : for (DerivationOutputs::iterator i = drv.outputs.begin();
1494 : i != drv.outputs.end(); ++i)
1495 : {
1496 113 : debug(format("building path `%1%'") % i->second.path);
1497 113 : allPaths.insert(i->second.path);
1498 : }
1499 :
1500 : /* Determine the full set of input paths. */
1501 :
1502 : /* First, the input derivations. */
1503 151 : for (DerivationInputs::iterator i = drv.inputDrvs.begin();
1504 : i != drv.inputDrvs.end(); ++i)
1505 : {
1506 : /* Add the relevant output closures of the input derivation
1507 : `*i' as input paths. Only add the closures of output paths
1508 : that are specified as inputs. */
1509 38 : assert(worker.store.isValidPath(i->first));
1510 38 : Derivation inDrv = derivationFromPath(i->first);
1511 152 : for (StringSet::iterator j = i->second.begin();
1512 : j != i->second.end(); ++j)
1513 38 : if (inDrv.outputs.find(*j) != inDrv.outputs.end())
1514 38 : computeFSClosure(inDrv.outputs[*j].path, inputPaths);
1515 : else
1516 : throw BuildError(
1517 : format("derivation `%1%' requires non-existent output `%2%' from input derivation `%3%'")
1518 0 : % drvPath % *j % i->first);
1519 : }
1520 :
1521 : /* Second, the input sources. */
1522 269 : for (PathSet::iterator i = drv.inputSrcs.begin();
1523 : i != drv.inputSrcs.end(); ++i)
1524 156 : computeFSClosure(*i, inputPaths);
1525 :
1526 113 : debug(format("added input paths %1%") % showPaths(inputPaths));
1527 :
1528 113 : allPaths.insert(inputPaths.begin(), inputPaths.end());
1529 :
1530 113 : return prProceed;
1531 : }
1532 :
1533 :
1534 112 : void DerivationGoal::startBuilder()
1535 : {
1536 112 : startNest(nest, lvlInfo,
1537 : format("building path(s) %1%") % showPaths(outputPaths(drv.outputs)))
1538 :
1539 : /* Right platform? */
1540 112 : if (drv.platform != thisSystem)
1541 : throw BuildError(
1542 : format("a `%1%' is required to build `%3%', but I am a `%2%'")
1543 0 : % drv.platform % thisSystem % drvPath);
1544 :
1545 : /* Construct the environment passed to the builder. */
1546 : typedef map<string, string> Environment;
1547 112 : Environment env;
1548 :
1549 : /* Most shells initialise PATH to some default (/bin:/usr/bin:...) when
1550 : PATH is not set. We don't want this, so we fill it in with some dummy
1551 : value. */
1552 112 : env["PATH"] = "/path-not-set";
1553 :
1554 : /* Set HOME to a non-existing path to prevent certain programs from using
1555 : /etc/passwd (or NIS, or whatever) to locate the home directory (for
1556 : example, wget looks for ~/.wgetrc). I.e., these tools use /etc/passwd
1557 : if HOME is not set, but they will just assume that the settings file
1558 : they are looking for does not exist if HOME is set but points to some
1559 : non-existing path. */
1560 224 : env["HOME"] = "/homeless-shelter";
1561 :
1562 : /* Tell the builder where the Nix store is. Usually they
1563 : shouldn't care, but this is useful for purity checking (e.g.,
1564 : the compiler or linker might only want to accept paths to files
1565 : in the store or in the build directory). */
1566 224 : env["NIX_STORE"] = nixStore;
1567 :
1568 : /* Add all bindings specified in the derivation. */
1569 1137 : for (StringPairs::iterator i = drv.env.begin();
1570 : i != drv.env.end(); ++i)
1571 913 : env[i->first] = i->second;
1572 :
1573 : /* Create a temporary directory where the build will take
1574 : place. */
1575 112 : tmpDir = createTempDir("", "nix-build-" + baseNameOf(drvPath), false, false);
1576 :
1577 : /* For convenience, set an environment pointing to the top build
1578 : directory. */
1579 112 : env["NIX_BUILD_TOP"] = tmpDir;
1580 :
1581 : /* Also set TMPDIR and variants to point to this directory. */
1582 224 : env["TMPDIR"] = env["TEMPDIR"] = env["TMP"] = env["TEMP"] = tmpDir;
1583 :
1584 : /* Explicitly set PWD to prevent problems with chroot builds. In
1585 : particular, dietlibc cannot figure out the cwd because the
1586 : inode of the current directory doesn't appear in .. (because
1587 : getdents returns the inode of the mount point). */
1588 224 : env["PWD"] = tmpDir;
1589 :
1590 : /* Compatibility hack with Nix <= 0.7: if this is a fixed-output
1591 : derivation, tell the builder, so that for instance `fetchurl'
1592 : can skip checking the output. On older Nixes, this environment
1593 : variable won't be set, so `fetchurl' will do the check. */
1594 112 : bool fixedOutput = true;
1595 336 : for (DerivationOutputs::iterator i = drv.outputs.begin();
1596 : i != drv.outputs.end(); ++i)
1597 112 : if (i->second.hash == "") fixedOutput = false;
1598 112 : if (fixedOutput)
1599 7 : env["NIX_OUTPUT_CHECKED"] = "1";
1600 :
1601 : /* *Only* if this is a fixed-output derivation, propagate the
1602 : values of the environment variables specified in the
1603 : `impureEnvVars' attribute to the builder. This allows for
1604 : instance environment variables for proxy configuration such as
1605 : `http_proxy' to be easily passed to downloaders like
1606 : `fetchurl'. Passing such environment variables from the caller
1607 : to the builder is generally impure, but the output of
1608 : fixed-output derivations is by definition pure (since we
1609 : already know the cryptographic hash of the output). */
1610 112 : if (fixedOutput) {
1611 7 : Strings varNames = tokenizeString(drv.env["impureEnvVars"]);
1612 49 : for (Strings::iterator i = varNames.begin(); i != varNames.end(); ++i)
1613 21 : env[*i] = getEnv(*i);
1614 : }
1615 :
1616 : /* The `exportReferencesGraph' feature allows the references graph
1617 : to be passed to a builder. This attribute should be a list of
1618 : pairs [name1 path1 name2 path2 ...]. The references graph of
1619 : each `pathN' will be stored in a text file `nameN' in the
1620 : temporary build directory. The text files have the format used
1621 : by `nix-store --register-validity'. However, the deriver
1622 : fields are left empty. */
1623 112 : string s = drv.env["exportReferencesGraph"];
1624 224 : Strings ss = tokenizeString(s);
1625 224 : if (ss.size() % 2 != 0)
1626 0 : throw BuildError(format("odd number of tokens in `exportReferencesGraph': `%1%'") % s);
1627 112 : for (Strings::iterator i = ss.begin(); i != ss.end(); ) {
1628 0 : string fileName = *i++;
1629 0 : checkStoreName(fileName); /* !!! abuse of this function */
1630 :
1631 : /* Check that the store path is valid. */
1632 0 : Path storePath = *i++;
1633 0 : if (!isInStore(storePath))
1634 : throw BuildError(format("`exportReferencesGraph' contains a non-store path `%1%'")
1635 0 : % storePath);
1636 0 : storePath = toStorePath(storePath);
1637 0 : if (!worker.store.isValidPath(storePath))
1638 : throw BuildError(format("`exportReferencesGraph' contains an invalid path `%1%'")
1639 0 : % storePath);
1640 :
1641 : /* Write closure info to `fileName'. */
1642 0 : PathSet refs;
1643 0 : computeFSClosure(storePath, refs);
1644 : /* !!! in secure Nix, the writing should be done on the
1645 : build uid for security (maybe). */
1646 : writeStringToFile(tmpDir + "/" + fileName,
1647 0 : makeValidityRegistration(refs, false, false));
1648 : }
1649 :
1650 : // The same for derivations
1651 : // !!! urgh, cut&paste duplication
1652 112 : s = drv.env["exportBuildReferencesGraph"];
1653 224 : ss = tokenizeString(s);
1654 224 : if (ss.size() % 2 != 0)
1655 0 : throw BuildError(format("odd number of tokens in `exportBuildReferencesGraph': `%1%'") % s);
1656 112 : for (Strings::iterator i = ss.begin(); i != ss.end(); ) {
1657 0 : string fileName = *i++;
1658 0 : checkStoreName(fileName); /* !!! abuse of this function */
1659 :
1660 : /* Check that the store path is valid. */
1661 0 : Path storePath = *i++;
1662 0 : if (!isInStore(storePath))
1663 : throw BuildError(format("`exportBuildReferencesGraph' contains a non-store path `%1%'")
1664 0 : % storePath);
1665 0 : storePath = toStorePath(storePath);
1666 0 : if (!worker.store.isValidPath(storePath))
1667 : throw BuildError(format("`exportBuildReferencesGraph' contains an invalid path `%1%'")
1668 0 : % storePath);
1669 :
1670 : /* Write closure info to `fileName'. */
1671 0 : PathSet refs1,refs;
1672 0 : computeFSClosure(storePath, refs1);
1673 0 : for (PathSet::iterator j = refs1.begin(); j != refs1.end() ; j++) {
1674 0 : refs.insert (*j);
1675 0 : if (isDerivation (*j)) {
1676 0 : Derivation deriv = derivationFromPath (*j);
1677 0 : for (DerivationOutputs::iterator k=deriv.outputs.begin();
1678 : k != deriv.outputs.end(); k++) {
1679 0 : refs.insert(k->second.path);
1680 :
1681 0 : }
1682 : }
1683 : }
1684 : /* !!! in secure Nix, the writing should be done on the
1685 : build uid for security (maybe). */
1686 : writeStringToFile(tmpDir + "/" + fileName,
1687 0 : makeValidityRegistration(refs, false, false));
1688 : }
1689 :
1690 :
1691 : /* If `build-users-group' is not empty, then we have to build as
1692 : one of the members of that group. */
1693 112 : if (haveBuildUsers()) {
1694 0 : buildUser.acquire();
1695 0 : assert(buildUser.getUID() != 0);
1696 0 : assert(buildUser.getGID() != 0);
1697 :
1698 : /* Make sure that no other processes are executing under this
1699 : uid. */
1700 0 : buildUser.kill();
1701 :
1702 : /* Change ownership of the temporary build directory, if we're
1703 : root. If we're not root, then the setuid helper will do it
1704 : just before it starts the builder. */
1705 0 : if (amPrivileged()) {
1706 0 : if (chown(tmpDir.c_str(), buildUser.getUID(), buildUser.getGID()) == -1)
1707 0 : throw SysError(format("cannot change ownership of `%1%'") % tmpDir);
1708 : }
1709 :
1710 : /* Check that the Nix store has the appropriate permissions,
1711 : i.e., owned by root and mode 1775 (sticky bit on so that
1712 : the builder can create its output but not mess with the
1713 : outputs of other processes). */
1714 : struct stat st;
1715 0 : if (stat(nixStore.c_str(), &st) == -1)
1716 0 : throw SysError(format("cannot stat `%1%'") % nixStore);
1717 0 : if (!(st.st_mode & S_ISVTX) ||
1718 : ((st.st_mode & S_IRWXG) != S_IRWXG) ||
1719 : (st.st_gid != buildUser.getGID()))
1720 : throw Error(format(
1721 : "builder does not have write permission to `%2%'; "
1722 : "try `chgrp %1% %2%; chmod 1775 %2%'")
1723 0 : % buildUser.getGID() % nixStore);
1724 : }
1725 :
1726 :
1727 : /* Are we doing a chroot build? Note that fixed-output
1728 : derivations are never done in a chroot, mainly so that
1729 : functions like fetchurl (which needs a proper /etc/resolv.conf)
1730 : work properly. Purity checking for fixed-output derivations
1731 : is somewhat pointless anyway. */
1732 112 : useChroot = queryBoolSetting("build-use-chroot", false);
1733 112 : Path chrootRootDir;
1734 :
1735 112 : if (fixedOutput) useChroot = false;
1736 :
1737 112 : if (useChroot) {
1738 : #if CHROOT_ENABLED
1739 : /* Create a temporary directory in which we set up the chroot
1740 : environment using bind-mounts.
1741 :
1742 : !!! Bind mounts are potentially dangerous: if the user
1743 : cleans up his system by doing "rm -rf
1744 : /nix/var/nix/chroots/*", this will recurse into /nix/store
1745 : via the bind mounts (and potentially other parts of the
1746 : filesystem, depending on the setting of the
1747 : `build-chroot-dirs' option). */
1748 0 : chrootRootDir = createTempDir(nixChrootsDir, "chroot-nix");
1749 :
1750 : /* Clean up the chroot directory automatically, but don't
1751 : recurse; that would be very very bad if the unmount of a
1752 : bind-mount fails. Instead BindMount::unbind() unmounts and
1753 : deletes exactly those directories that it created to
1754 : produce the mount point, so that after all the BindMount
1755 : destructors have run, chrootRootDir should be empty. */
1756 0 : autoDelChrootRoot = boost::shared_ptr<AutoDelete>(new AutoDelete(chrootRootDir, false));
1757 :
1758 0 : printMsg(lvlChatty, format("setting up chroot environment in `%1%'") % chrootRootDir);
1759 :
1760 : /* Create a writable /tmp in the chroot. Many builders need
1761 : this. (Of course they should really respect $TMPDIR
1762 : instead.) */
1763 0 : Path chrootTmpDir = chrootRootDir + "/tmp";
1764 0 : createDirs(chrootTmpDir);
1765 :
1766 0 : if (chmod(chrootTmpDir.c_str(), 01777) == -1)
1767 0 : throw SysError("creating /tmp in the chroot");
1768 :
1769 : /* When deleting this, do recurse (the builder might have left
1770 : crap there). As long as nothing important is bind-mounted
1771 : under /tmp it's okay (and the bind-mounts are unmounted
1772 : before autoDelChrootTmp's destructor runs, anyway). */
1773 0 : autoDelChrootTmp = boost::shared_ptr<AutoDelete>(new AutoDelete(chrootTmpDir));
1774 :
1775 : /* Bind-mount a user-configurable set of directories from the
1776 : host file system. The `/dev/pts' directory must be mounted
1777 : separately so that newly-created pseudo-terminals show
1778 : up. */
1779 0 : Paths defaultDirs;
1780 0 : defaultDirs.push_back("/dev");
1781 0 : defaultDirs.push_back("/dev/pts");
1782 0 : defaultDirs.push_back("/proc");
1783 :
1784 0 : Paths dirsInChroot = querySetting("build-chroot-dirs", defaultDirs);
1785 :
1786 0 : dirsInChroot.push_front(nixStore);
1787 0 : dirsInChroot.push_front(tmpDir);
1788 :
1789 : /* Push BindMounts at the front of the list so that they get
1790 : unmounted in LIFO order. (!!! Does the C++ standard
1791 : guarantee that list elements are destroyed in order?) */
1792 0 : for (Paths::iterator i = dirsInChroot.begin(); i != dirsInChroot.end(); ++i)
1793 0 : bindMounts.push_front(boost::shared_ptr<BindMount>(new BindMount(*i, chrootRootDir + *i)));
1794 :
1795 : #else
1796 : throw Error("chroot builds are not supported on this platform");
1797 : #endif
1798 : }
1799 :
1800 :
1801 : /* Run the builder. */
1802 112 : printMsg(lvlChatty, format("executing builder `%1%'") %
1803 : drv.builder);
1804 :
1805 : /* Create the log file and pipe. */
1806 112 : Path logFile = openLogFile();
1807 :
1808 : /* Fork a child to build the package. Note that while we
1809 : currently use forks to run and wait for the children, it
1810 : shouldn't be hard to use threads for this on systems where
1811 : fork() is unavailable or inefficient. */
1812 112 : pid = fork();
1813 112 : switch (pid) {
1814 :
1815 : case -1:
1816 0 : throw SysError("unable to fork");
1817 :
1818 : case 0:
1819 :
1820 : /* Warning: in the child we should absolutely not make any
1821 : Berkeley DB calls! */
1822 :
1823 : try { /* child */
1824 :
1825 : #if CHROOT_ENABLED
1826 : /* If building in a chroot, do the chroot right away.
1827 : initChild() will do a chdir() to the temporary build
1828 : directory to make sure the current directory is in the
1829 : chroot. (Actually the order doesn't matter, since due
1830 : to the bind mount tmpDir and tmpRootDit/tmpDir are the
1831 : same directories.) */
1832 0 : if (useChroot && chroot(chrootRootDir.c_str()) == -1)
1833 0 : throw SysError(format("cannot change root directory to `%1%'") % chrootRootDir);
1834 : #endif
1835 :
1836 0 : initChild();
1837 :
1838 : /* Fill in the environment. */
1839 0 : Strings envStrs;
1840 0 : for (Environment::const_iterator i = env.begin();
1841 : i != env.end(); ++i)
1842 0 : envStrs.push_back(i->first + "=" + i->second);
1843 0 : const char * * envArr = strings2CharPtrs(envStrs);
1844 :
1845 0 : Path program = drv.builder.c_str();
1846 0 : std::vector<const char *> args; /* careful with c_str()! */
1847 0 : string user; /* must be here for its c_str()! */
1848 :
1849 : /* If we are running in `build-users' mode, then switch to
1850 : the user we allocated above. Make sure that we drop
1851 : all root privileges. Note that initChild() above has
1852 : closed all file descriptors except std*, so that's
1853 : safe. Also note that setuid() when run as root sets
1854 : the real, effective and saved UIDs. */
1855 0 : if (buildUser.enabled()) {
1856 0 : printMsg(lvlChatty, format("switching to user `%1%'") % buildUser.getUser());
1857 :
1858 0 : if (amPrivileged()) {
1859 :
1860 0 : if (setgroups(0, 0) == -1)
1861 0 : throw SysError("cannot clear the set of supplementary groups");
1862 :
1863 0 : if (setgid(buildUser.getGID()) == -1 ||
1864 : getgid() != buildUser.getGID() ||
1865 : getegid() != buildUser.getGID())
1866 0 : throw SysError("setgid failed");
1867 :
1868 0 : if (setuid(buildUser.getUID()) == -1 ||
1869 : getuid() != buildUser.getUID() ||
1870 : geteuid() != buildUser.getUID())
1871 0 : throw SysError("setuid failed");
1872 :
1873 : } else {
1874 : /* Let the setuid helper take care of it. */
1875 0 : program = nixLibexecDir + "/nix-setuid-helper";
1876 0 : args.push_back(program.c_str());
1877 0 : args.push_back("run-builder");
1878 0 : user = buildUser.getUser().c_str();
1879 0 : args.push_back(user.c_str());
1880 0 : args.push_back(drv.builder.c_str());
1881 : }
1882 : }
1883 :
1884 : /* Fill in the arguments. */
1885 0 : string builderBasename = baseNameOf(drv.builder);
1886 0 : args.push_back(builderBasename.c_str());
1887 0 : for (Strings::iterator i = drv.args.begin();
1888 : i != drv.args.end(); ++i)
1889 0 : args.push_back(i->c_str());
1890 0 : args.push_back(0);
1891 :
1892 0 : restoreSIGPIPE();
1893 :
1894 : /* Execute the program. This should not return. */
1895 0 : execve(program.c_str(), (char * *) &args[0], (char * *) envArr);
1896 :
1897 : throw SysError(format("executing `%1%'")
1898 0 : % drv.builder);
1899 :
1900 0 : } catch (std::exception & e) {
1901 0 : std::cerr << format("build error: %1%") % e.what() << std::endl;
1902 : }
1903 0 : quickExit(1);
1904 : }
1905 :
1906 :
1907 : /* parent */
1908 112 : pid.setSeparatePG(true);
1909 112 : logPipe.writeSide.close();
1910 : worker.childStarted(shared_from_this(), pid,
1911 112 : singleton<set<int> >(logPipe.readSide), true);
1912 :
1913 112 : if (printBuildTrace) {
1914 0 : printMsg(lvlError, format("@ build-started %1% %2% %3% %4%")
1915 : % drvPath % drv.outputs["out"].path % drv.platform % logFile);
1916 112 : }
1917 112 : }
1918 :
1919 :
1920 : /* Parse a list of reference specifiers. Each element must either be
1921 : a store path, or the symbolic name of the output of the derivation
1922 : (such as `out'). */
1923 5 : PathSet parseReferenceSpecifiers(const Derivation & drv, string attr)
1924 : {
1925 5 : PathSet result;
1926 5 : Paths paths = tokenizeString(attr);
1927 12 : for (Strings::iterator i = paths.begin(); i != paths.end(); ++i) {
1928 2 : if (isStorePath(*i))
1929 1 : result.insert(*i);
1930 1 : else if (drv.outputs.find(*i) != drv.outputs.end())
1931 1 : result.insert(drv.outputs.find(*i)->second.path);
1932 : else throw BuildError(
1933 : format("derivation contains an illegal reference specifier `%1%'")
1934 0 : % *i);
1935 : }
1936 5 : return result;
1937 : }
1938 :
1939 :
1940 110 : void DerivationGoal::computeClosure()
1941 : {
1942 110 : map<Path, PathSet> allReferences;
1943 110 : map<Path, Hash> contentHashes;
1944 :
1945 : /* Check whether the output paths were created, and grep each
1946 : output path to determine what other paths it references. Also make all
1947 : output paths read-only. */
1948 217 : for (DerivationOutputs::iterator i = drv.outputs.begin();
1949 : i != drv.outputs.end(); ++i)
1950 : {
1951 110 : Path path = i->second.path;
1952 110 : if (!pathExists(path)) {
1953 : throw BuildError(
1954 : format("builder for `%1%' failed to produce output path `%2%'")
1955 0 : % drvPath % path);
1956 : }
1957 :
1958 : struct stat st;
1959 110 : if (lstat(path.c_str(), &st) == -1)
1960 0 : throw SysError(format("getting attributes of path `%1%'") % path);
1961 :
1962 110 : startNest(nest, lvlTalkative,
1963 : format("scanning for references inside `%1%'") % path);
1964 :
1965 : /* Check that fixed-output derivations produced the right
1966 : outputs (i.e., the content hash should match the specified
1967 : hash). */
1968 110 : if (i->second.hash != "") {
1969 :
1970 7 : bool recursive = false;
1971 7 : string algo = i->second.hashAlgo;
1972 :
1973 7 : if (string(algo, 0, 2) == "r:") {
1974 4 : recursive = true;
1975 4 : algo = string(algo, 2);
1976 : }
1977 :
1978 7 : if (!recursive) {
1979 : /* The output path should be a regular file without
1980 : execute permission. */
1981 3 : if (!S_ISREG(st.st_mode) || (st.st_mode & S_IXUSR) != 0)
1982 : throw BuildError(
1983 : format("output path `%1% should be a non-executable regular file")
1984 0 : % path);
1985 : }
1986 :
1987 : /* Check the hash. */
1988 7 : HashType ht = parseHashType(algo);
1989 7 : if (ht == htUnknown)
1990 0 : throw BuildError(format("unknown hash algorithm `%1%'") % algo);
1991 7 : Hash h = parseHash(ht, i->second.hash);
1992 7 : Hash h2 = recursive ? hashPath(ht, path) : hashFile(ht, path);
1993 7 : if (h != h2)
1994 : throw BuildError(
1995 : format("output path `%1%' should have %2% hash `%3%', instead has `%4%'")
1996 1 : % path % algo % printHash(h) % printHash(h2));
1997 : }
1998 :
1999 : /* Get rid of all weird permissions. */
2000 109 : canonicalisePathMetaData(path);
2001 :
2002 : /* For this output path, find the references to other paths contained
2003 : in it. */
2004 109 : PathSet references = scanForReferences(path, allPaths);
2005 :
2006 : /* For debugging, print out the referenced and unreferenced
2007 : paths. */
2008 339 : for (PathSet::iterator i = inputPaths.begin();
2009 : i != inputPaths.end(); ++i)
2010 : {
2011 230 : PathSet::iterator j = references.find(*i);
2012 230 : if (j == references.end())
2013 153 : debug(format("unreferenced input: `%1%'") % *i);
2014 : else
2015 77 : debug(format("referenced input: `%1%'") % *i);
2016 : }
2017 :
2018 109 : allReferences[path] = references;
2019 :
2020 : /* If the derivation specifies an `allowedReferences'
2021 : attribute (containing a list of paths that the output may
2022 : refer to), check that all references are in that list. !!!
2023 : allowedReferences should really be per-output. */
2024 109 : if (drv.env.find("allowedReferences") != drv.env.end()) {
2025 5 : PathSet allowed = parseReferenceSpecifiers(drv, drv.env["allowedReferences"]);
2026 12 : for (PathSet::iterator i = references.begin(); i != references.end(); ++i)
2027 4 : if (allowed.find(*i) == allowed.end())
2028 5 : throw BuildError(format("output is not allowed to refer to path `%1%'") % *i);
2029 : }
2030 :
2031 : /* Hash the contents of the path. The hash is stored in the
2032 : database so that we can verify later on whether nobody has
2033 : messed with the store. !!! inefficient: it would be nice
2034 : if we could combine this with filterReferences(). */
2035 107 : contentHashes[path] = hashPath(htSHA256, path);
2036 : }
2037 :
2038 : /* Register each output path as valid, and register the sets of
2039 : paths referenced by each of them. !!! this should be
2040 : atomic so that either all paths are registered as valid, or
2041 : none are. */
2042 214 : for (DerivationOutputs::iterator i = drv.outputs.begin();
2043 : i != drv.outputs.end(); ++i)
2044 : {
2045 : worker.store.registerValidPath(i->second.path,
2046 : contentHashes[i->second.path],
2047 : allReferences[i->second.path],
2048 107 : drvPath);
2049 : }
2050 :
2051 : /* It is now safe to delete the lock files, since all future
2052 : lockers will see that the output paths are valid; they will not
2053 : create new lock files with the same names as the old (unlinked)
2054 : lock files. */
2055 110 : outputLocks.setDeletion(true);
2056 107 : }
2057 :
2058 :
2059 1704 : string drvsLogDir = "drvs";
2060 :
2061 :
2062 115 : Path DerivationGoal::openLogFile()
2063 : {
2064 : /* Create a log file. */
2065 115 : Path dir = (format("%1%/%2%") % nixLogDir % drvsLogDir).str();
2066 230 : createDirs(dir);
2067 :
2068 115 : Path logFileName = (format("%1%/%2%") % dir % baseNameOf(drvPath)).str();
2069 : fdLogFile = open(logFileName.c_str(),
2070 115 : O_CREAT | O_WRONLY | O_TRUNC, 0666);
2071 115 : if (fdLogFile == -1)
2072 0 : throw SysError(format("creating log file `%1%'") % logFileName);
2073 :
2074 : /* Create a pipe to get the output of the child. */
2075 115 : logPipe.create();
2076 :
2077 115 : return logFileName;
2078 : }
2079 :
2080 :
2081 3 : void DerivationGoal::initChild()
2082 : {
2083 3 : commonChildInit(logPipe);
2084 :
2085 3 : if (chdir(tmpDir.c_str()) == -1)
2086 0 : throw SysError(format("changing into `%1%'") % tmpDir);
2087 :
2088 : /* When running a hook, dup the communication pipes. */
2089 3 : bool inHook = fromHook.writeSide.isOpen();
2090 3 : if (inHook) {
2091 3 : fromHook.readSide.close();
2092 3 : if (dup2(fromHook.writeSide, 3) == -1)
2093 0 : throw SysError("dupping from-hook write side");
2094 :
2095 3 : toHook.writeSide.close();
2096 3 : if (dup2(toHook.readSide, 4) == -1)
2097 0 : throw SysError("dupping to-hook read side");
2098 : }
2099 :
2100 : /* Close all other file descriptors. */
2101 3 : set<int> exceptions;
2102 3 : if (inHook) {
2103 3 : exceptions.insert(3);
2104 3 : exceptions.insert(4);
2105 : }
2106 3 : closeMostFDs(exceptions);
2107 3 : }
2108 :
2109 :
2110 335 : void DerivationGoal::deleteTmpDir(bool force)
2111 : {
2112 335 : if (tmpDir != "") {
2113 115 : if (keepFailed && !force) {
2114 0 : printMsg(lvlError,
2115 : format("builder for `%1%' failed; keeping build directory `%2%'")
2116 : % drvPath % tmpDir);
2117 0 : if (buildUser.enabled() && !amPrivileged())
2118 0 : getOwnership(tmpDir);
2119 : }
2120 : else
2121 115 : deletePathWrapped(tmpDir);
2122 115 : tmpDir = "";
2123 : }
2124 335 : }
2125 :
2126 :
2127 352 : void DerivationGoal::handleChildOutput(int fd, const string & data)
2128 : {
2129 352 : if (fd == logPipe.readSide) {
2130 352 : if (verbosity >= buildVerbosity)
2131 352 : writeToStderr((unsigned char *) data.c_str(), data.size());
2132 352 : writeFull(fdLogFile, (unsigned char *) data.c_str(), data.size());
2133 : }
2134 :
2135 0 : else abort();
2136 352 : }
2137 :
2138 :
2139 113 : void DerivationGoal::handleEOF(int fd)
2140 : {
2141 113 : if (fd == logPipe.readSide) worker.wakeUp(shared_from_this());
2142 113 : }
2143 :
2144 :
2145 640 : PathSet DerivationGoal::checkPathValidity(bool returnValid)
2146 : {
2147 640 : PathSet result;
2148 1280 : for (DerivationOutputs::iterator i = drv.outputs.begin();
2149 : i != drv.outputs.end(); ++i)
2150 640 : if (worker.store.isValidPath(i->second.path)) {
2151 106 : if (returnValid) result.insert(i->second.path);
2152 : } else {
2153 534 : if (!returnValid) result.insert(i->second.path);
2154 : }
2155 0 : return result;
2156 : }
2157 :
2158 :
2159 :
2160 : //////////////////////////////////////////////////////////////////////
2161 :
2162 :
2163 : class SubstitutionGoal : public Goal
2164 : {
2165 : friend class Worker;
2166 :
2167 : private:
2168 : /* The store path that should be realised through a substitute. */
2169 : Path storePath;
2170 :
2171 : /* The remaining substituters. */
2172 : Paths subs;
2173 :
2174 : /* The current substituter. */
2175 : Path sub;
2176 :
2177 : /* Path info returned by the substituter's query info operation. */
2178 : SubstitutablePathInfo info;
2179 :
2180 : /* Pipe for the substitute's standard output/error. */
2181 : Pipe logPipe;
2182 :
2183 : /* The process ID of the builder. */
2184 : Pid pid;
2185 :
2186 : /* Lock on the store path. */
2187 : boost::shared_ptr<PathLocks> outputLock;
2188 :
2189 : typedef void (SubstitutionGoal::*GoalState)();
2190 : GoalState state;
2191 :
2192 : public:
2193 : SubstitutionGoal(const Path & storePath, Worker & worker);
2194 : ~SubstitutionGoal();
2195 :
2196 : void cancel();
2197 :
2198 : void work();
2199 :
2200 : /* The states. */
2201 : void init();
2202 : void tryNext();
2203 : void gotInfo();
2204 : void referencesValid();
2205 : void tryToRun();
2206 : void finished();
2207 :
2208 : /* Callback used by the worker to write to the log. */
2209 : void handleChildOutput(int fd, const string & data);
2210 : void handleEOF(int fd);
2211 : };
2212 :
2213 :
2214 391 : SubstitutionGoal::SubstitutionGoal(const Path & storePath, Worker & worker)
2215 391 : : Goal(worker)
2216 : {
2217 391 : this->storePath = storePath;
2218 391 : state = &SubstitutionGoal::init;
2219 391 : name = (format("substitution of `%1%'") % storePath).str();
2220 391 : trace("created");
2221 391 : }
2222 :
2223 :
2224 391 : SubstitutionGoal::~SubstitutionGoal()
2225 : {
2226 : /* !!! Once we let substitution goals run under a build user, we
2227 : need to do use the setuid helper just as in ~DerivationGoal().
2228 : Idem for cancel. */
2229 391 : if (pid != -1) worker.childTerminated(pid);
2230 391 : }
2231 :
2232 :
2233 0 : void SubstitutionGoal::cancel()
2234 : {
2235 0 : if (pid != -1) {
2236 0 : pid_t savedPid = pid;
2237 0 : pid.kill();
2238 0 : worker.childTerminated(savedPid);
2239 : }
2240 0 : amDone(ecFailed);
2241 0 : }
2242 :
2243 :
2244 437 : void SubstitutionGoal::work()
2245 : {
2246 437 : (this->*state)();
2247 437 : }
2248 :
2249 :
2250 391 : void SubstitutionGoal::init()
2251 : {
2252 391 : trace("init");
2253 :
2254 391 : worker.store.addTempRoot(storePath);
2255 :
2256 : /* If the path already exists we're done. */
2257 391 : if (worker.store.isValidPath(storePath)) {
2258 375 : amDone(ecSuccess);
2259 375 : return;
2260 : }
2261 :
2262 16 : subs = substituters;
2263 :
2264 16 : tryNext();
2265 : }
2266 :
2267 :
2268 31 : void SubstitutionGoal::tryNext()
2269 : {
2270 31 : trace("trying next substituter");
2271 :
2272 31 : if (subs.size() == 0) {
2273 : /* None left. Terminate this goal and let someone else deal
2274 : with it. */
2275 2 : printMsg(lvlError,
2276 : format("path `%1%' is required, but there is no substituter that can build it")
2277 : % storePath);
2278 2 : amDone(ecFailed);
2279 2 : return;
2280 : }
2281 :
2282 29 : sub = subs.front();
2283 29 : subs.pop_front();
2284 :
2285 29 : if (!worker.store.querySubstitutablePathInfo(sub, storePath, info)) {
2286 12 : tryNext();
2287 12 : return;
2288 : }
2289 :
2290 : /* To maintain the closure invariant, we first have to realise the
2291 : paths referenced by this one. */
2292 28 : foreach (PathSet::iterator, i, info.references)
2293 11 : if (*i != storePath) /* ignore self-references */
2294 8 : addWaitee(worker.makeSubstitutionGoal(*i));
2295 :
2296 17 : if (waitees.empty()) /* to prevent hang (no wake-up event) */
2297 11 : referencesValid();
2298 : else
2299 6 : state = &SubstitutionGoal::referencesValid;
2300 : }
2301 :
2302 :
2303 17 : void SubstitutionGoal::referencesValid()
2304 : {
2305 17 : trace("all references realised");
2306 :
2307 17 : if (nrFailed > 0) {
2308 0 : printMsg(lvlError,
2309 : format("some references of path `%1%' could not be realised") % storePath);
2310 0 : amDone(ecFailed);
2311 0 : return;
2312 : }
2313 :
2314 28 : foreach (PathSet::iterator, i, info.references)
2315 11 : if (*i != storePath) /* ignore self-references */
2316 8 : assert(worker.store.isValidPath(*i));
2317 :
2318 17 : state = &SubstitutionGoal::tryToRun;
2319 17 : worker.waitForBuildSlot(shared_from_this());
2320 : }
2321 :
2322 :
2323 20 : void SubstitutionGoal::tryToRun()
2324 : {
2325 20 : trace("trying to run");
2326 :
2327 : /* Make sure that we are allowed to start a build. */
2328 20 : if (!worker.canBuildMore()) {
2329 3 : worker.waitForBuildSlot(shared_from_this());
2330 3 : return;
2331 : }
2332 :
2333 : /* Maybe a derivation goal has already locked this path
2334 : (exceedingly unlikely, since it should have used a substitute
2335 : first, but let's be defensive). */
2336 17 : outputLock.reset(); // make sure this goal's lock is gone
2337 17 : if (pathIsLockedByMe(storePath)) {
2338 0 : debug(format("restarting substitution of `%1%' because it's locked by another goal")
2339 : % storePath);
2340 0 : worker.waitForAnyGoal(shared_from_this());
2341 0 : return; /* restart in the tryToRun() state when another goal finishes */
2342 : }
2343 :
2344 : /* Acquire a lock on the output path. */
2345 17 : outputLock = boost::shared_ptr<PathLocks>(new PathLocks);
2346 : outputLock->lockPaths(singleton<PathSet>(storePath),
2347 34 : (format("waiting for lock on `%1%'") % storePath).str());
2348 :
2349 : /* Check again whether the path is invalid. */
2350 17 : if (worker.store.isValidPath(storePath)) {
2351 0 : debug(format("store path `%1%' has become valid") % storePath);
2352 0 : outputLock->setDeletion(true);
2353 0 : amDone(ecSuccess);
2354 0 : return;
2355 : }
2356 :
2357 17 : printMsg(lvlInfo,
2358 : format("substituting path `%1%' using substituter `%2%'")
2359 : % storePath % sub);
2360 :
2361 17 : logPipe.create();
2362 :
2363 : /* Remove the (stale) output path if it exists. */
2364 17 : if (pathExists(storePath))
2365 0 : deletePathWrapped(storePath);
2366 :
2367 : /* Fork the substitute program. */
2368 17 : pid = fork();
2369 17 : switch (pid) {
2370 :
2371 : case -1:
2372 0 : throw SysError("unable to fork");
2373 :
2374 : case 0:
2375 : try { /* child */
2376 :
2377 0 : logPipe.readSide.close();
2378 :
2379 : /* !!! close other handles */
2380 :
2381 0 : commonChildInit(logPipe);
2382 :
2383 : /* Fill in the arguments. */
2384 0 : Strings args;
2385 0 : args.push_back(baseNameOf(sub));
2386 0 : args.push_back("--substitute");
2387 0 : args.push_back(storePath);
2388 0 : const char * * argArr = strings2CharPtrs(args);
2389 :
2390 0 : execv(sub.c_str(), (char * *) argArr);
2391 :
2392 0 : throw SysError(format("executing `%1%'") % sub);
2393 :
2394 0 : } catch (std::exception & e) {
2395 0 : std::cerr << format("substitute error: %1%") % e.what() << std::endl;
2396 : }
2397 0 : quickExit(1);
2398 : }
2399 :
2400 : /* parent */
2401 17 : pid.setSeparatePG(true);
2402 17 : pid.setKillSignal(SIGTERM);
2403 17 : logPipe.writeSide.close();
2404 : worker.childStarted(shared_from_this(),
2405 17 : pid, singleton<set<int> >(logPipe.readSide), true);
2406 :
2407 17 : state = &SubstitutionGoal::finished;
2408 :
2409 17 : if (printBuildTrace) {
2410 0 : printMsg(lvlError, format("@ substituter-started %1% %2%")
2411 : % storePath % sub);
2412 : }
2413 : }
2414 :
2415 :
2416 17 : void SubstitutionGoal::finished()
2417 : {
2418 17 : trace("substitute finished");
2419 :
2420 : /* Since we got an EOF on the logger pipe, the substitute is
2421 : presumed to have terminated. */
2422 : /* !!! this could block! */
2423 17 : pid_t savedPid = pid;
2424 17 : int status = pid.wait(true);
2425 :
2426 : /* So the child is gone now. */
2427 17 : worker.childTerminated(savedPid);
2428 :
2429 : /* Close the read side of the logger pipe. */
2430 17 : logPipe.readSide.close();
2431 :
2432 17 : debug(format("substitute for `%1%' finished") % storePath);
2433 :
2434 : /* Check the exit status and the build result. */
2435 : try {
2436 :
2437 17 : if (!statusOk(status))
2438 : throw SubstError(format("builder for `%1%' %2%")
2439 3 : % storePath % statusToString(status));
2440 :
2441 14 : if (!pathExists(storePath))
2442 : throw SubstError(
2443 : format("substitute did not produce path `%1%'")
2444 0 : % storePath);
2445 :
2446 6 : } catch (SubstError & e) {
2447 :
2448 3 : printMsg(lvlInfo,
2449 : format("substitution of path `%1%' using substituter `%2%' failed: %3%")
2450 : % storePath % sub % e.msg());
2451 :
2452 3 : if (printBuildTrace) {
2453 0 : printMsg(lvlError, format("@ substituter-failed %1% %2% %3%")
2454 : % storePath % status % e.msg());
2455 : }
2456 :
2457 : /* Try the next substitute. */
2458 3 : state = &SubstitutionGoal::tryNext;
2459 3 : worker.wakeUp(shared_from_this());
2460 3 : return;
2461 : }
2462 :
2463 14 : canonicalisePathMetaData(storePath);
2464 :
2465 14 : Hash contentHash = hashPath(htSHA256, storePath);
2466 :
2467 : worker.store.registerValidPath(storePath, contentHash,
2468 14 : info.references, info.deriver);
2469 :
2470 14 : outputLock->setDeletion(true);
2471 :
2472 16 : printMsg(lvlChatty,
2473 : format("substitution of path `%1%' succeeded") % storePath);
2474 :
2475 14 : if (printBuildTrace) {
2476 0 : printMsg(lvlError, format("@ substituter-succeeded %1%") % storePath);
2477 : }
2478 :
2479 14 : amDone(ecSuccess);
2480 : }
2481 :
2482 :
2483 2465 : void SubstitutionGoal::handleChildOutput(int fd, const string & data)
2484 : {
2485 2465 : assert(fd == logPipe.readSide);
2486 2465 : if (verbosity >= buildVerbosity)
2487 2465 : writeToStderr((unsigned char *) data.c_str(), data.size());
2488 : /* Don't write substitution output to a log file for now. We
2489 : probably should, though. */
2490 2465 : }
2491 :
2492 :
2493 17 : void SubstitutionGoal::handleEOF(int fd)
2494 : {
2495 17 : if (fd == logPipe.readSide) worker.wakeUp(shared_from_this());
2496 17 : }
2497 :
2498 :
2499 :
2500 : //////////////////////////////////////////////////////////////////////
2501 :
2502 :
2503 : static bool working = false;
2504 :
2505 :
2506 174 : Worker::Worker(LocalStore & store)
2507 174 : : store(store)
2508 : {
2509 : /* Debugging: prevent recursive workers. */
2510 174 : if (working) abort();
2511 174 : working = true;
2512 174 : nrChildren = 0;
2513 174 : }
2514 :
2515 :
2516 174 : Worker::~Worker()
2517 : {
2518 174 : working = false;
2519 :
2520 : /* Explicitly get rid of all strong pointers now. After this all
2521 : goals that refer to this worker should be gone. (Otherwise we
2522 : are in trouble, since goals may call childTerminated() etc. in
2523 : their destructors). */
2524 174 : topGoals.clear();
2525 174 : }
2526 :
2527 :
2528 : template<class T>
2529 : static GoalPtr addGoal(const Path & path,
2530 651 : Worker & worker, WeakGoalMap & goalMap)
2531 : {
2532 651 : GoalPtr goal = goalMap[path].lock();
2533 651 : if (!goal) {
2534 611 : goal = GoalPtr(new T(path, worker));
2535 611 : goalMap[path] = goal;
2536 611 : worker.wakeUp(goal);
2537 : }
2538 0 : return goal;
2539 : }
2540 :
2541 :
2542 234 : GoalPtr Worker::makeDerivationGoal(const Path & nePath)
2543 : {
2544 234 : return addGoal<DerivationGoal>(nePath, *this, derivationGoals);
2545 : }
2546 :
2547 :
2548 417 : GoalPtr Worker::makeSubstitutionGoal(const Path & storePath)
2549 : {
2550 417 : return addGoal<SubstitutionGoal>(storePath, *this, substitutionGoals);
2551 : }
2552 :
2553 :
2554 1220 : static void removeGoal(GoalPtr goal, WeakGoalMap & goalMap)
2555 : {
2556 : /* !!! inefficient */
2557 4296 : for (WeakGoalMap::iterator i = goalMap.begin();
2558 : i != goalMap.end(); )
2559 1856 : if (i->second.lock() == goal) {
2560 610 : WeakGoalMap::iterator j = i; ++j;
2561 610 : goalMap.erase(i);
2562 610 : i = j;
2563 : }
2564 1246 : else ++i;
2565 1220 : }
2566 :
2567 :
2568 610 : void Worker::removeGoal(GoalPtr goal)
2569 : {
2570 610 : nix::removeGoal(goal, derivationGoals);
2571 1220 : nix::removeGoal(goal, substitutionGoals);
2572 610 : if (topGoals.find(goal) != topGoals.end()) {
2573 167 : topGoals.erase(goal);
2574 : /* If a top-level goal failed, then kill all other goals
2575 : (unless keepGoing was set). */
2576 167 : if (goal->getExitCode() == Goal::ecFailed && !keepGoing)
2577 6 : topGoals.clear();
2578 : }
2579 :
2580 : /* Wake up goals waiting for any goal to finish. */
2581 611 : for (WeakGoals::iterator i = waitingForAnyGoal.begin();
2582 : i != waitingForAnyGoal.end(); ++i)
2583 : {
2584 1 : GoalPtr goal = i->lock();
2585 1 : if (goal) wakeUp(goal);
2586 : }
2587 :
2588 610 : waitingForAnyGoal.clear();
2589 610 : }
2590 :
2591 :
2592 1327 : void Worker::wakeUp(GoalPtr goal)
2593 : {
2594 1327 : goal->trace("woken up");
2595 2654 : awake.insert(goal);
2596 1327 : }
2597 :
2598 :
2599 287 : bool Worker::canBuildMore()
2600 : {
2601 287 : return nrChildren < maxBuildJobs;
2602 : }
2603 :
2604 :
2605 : void Worker::childStarted(GoalPtr goal,
2606 132 : pid_t pid, const set<int> & fds, bool inBuildSlot)
2607 : {
2608 132 : Child child;
2609 132 : child.goal = goal;
2610 132 : child.fds = fds;
2611 132 : child.lastOutput = time(0);
2612 132 : child.inBuildSlot = inBuildSlot;
2613 132 : children[pid] = child;
2614 132 : if (inBuildSlot) nrChildren++;
2615 132 : }
2616 :
2617 :
2618 132 : void Worker::childTerminated(pid_t pid, bool wakeSleepers)
2619 : {
2620 132 : assert(pid != -1); /* common mistake */
2621 :
2622 132 : Children::iterator i = children.find(pid);
2623 132 : assert(i != children.end());
2624 :
2625 132 : if (i->second.inBuildSlot) {
2626 129 : assert(nrChildren > 0);
2627 129 : nrChildren--;
2628 : }
2629 :
2630 132 : children.erase(pid);
2631 :
2632 132 : if (wakeSleepers) {
2633 :
2634 : /* Wake up goals waiting for a build slot. */
2635 188 : for (WeakGoals::iterator i = wantingToBuild.begin();
2636 : i != wantingToBuild.end(); ++i)
2637 : {
2638 58 : GoalPtr goal = i->lock();
2639 58 : if (goal) wakeUp(goal);
2640 : }
2641 :
2642 130 : wantingToBuild.clear();
2643 : }
2644 132 : }
2645 :
2646 :
2647 73 : void Worker::waitForBuildSlot(GoalPtr goal)
2648 : {
2649 73 : debug("wait for build slot");
2650 73 : if (canBuildMore())
2651 15 : wakeUp(goal); /* we can do it right away */
2652 : else
2653 58 : wantingToBuild.insert(goal);
2654 73 : }
2655 :
2656 :
2657 0 : void Worker::waitForChildTermination(GoalPtr goal)
2658 : {
2659 0 : debug("wait for child termination");
2660 0 : if (children.size() == 0)
2661 : throw Error("waiting for a build slot, yet there are no running children - "
2662 0 : "maybe the build hook gave an inappropriate `postpone' reply?");
2663 0 : wantingToBuild.insert(goal);
2664 0 : }
2665 :
2666 :
2667 1 : void Worker::waitForAnyGoal(GoalPtr goal)
2668 : {
2669 1 : debug("wait for any goal");
2670 1 : waitingForAnyGoal.insert(goal);
2671 1 : }
2672 :
2673 :
2674 174 : void Worker::run(const Goals & _topGoals)
2675 : {
2676 342 : for (Goals::iterator i = _topGoals.begin();
2677 : i != _topGoals.end(); ++i)
2678 168 : topGoals.insert(*i);
2679 :
2680 174 : startNest(nest, lvlDebug, format("entered goal loop"));
2681 :
2682 2945 : while (1) {
2683 :
2684 3119 : checkInterrupt();
2685 :
2686 : /* Call every wake goal. */
2687 4138 : while (!awake.empty() && !topGoals.empty()) {
2688 1020 : WeakGoals awake2(awake);
2689 1020 : awake.clear();
2690 2199 : for (WeakGoals::iterator i = awake2.begin(); i != awake2.end(); ++i) {
2691 1327 : checkInterrupt();
2692 1327 : GoalPtr goal = i->lock();
2693 1327 : if (goal) goal->work();
2694 1474 : if (topGoals.empty()) break;
2695 : }
2696 : }
2697 :
2698 3118 : if (topGoals.empty()) break;
2699 :
2700 : /* Wait for input. */
2701 2945 : if (!children.empty())
2702 2945 : waitForInput();
2703 : else
2704 : /* !!! not when we're polling */
2705 0 : assert(!awake.empty());
2706 : }
2707 :
2708 : /* If --keep-going is not set, it's possible that the main goal
2709 : exited while some of its subgoals were still active. But if
2710 : --keep-going *is* set, then they must all be finished now. */
2711 173 : assert(!keepGoing || awake.empty());
2712 173 : assert(!keepGoing || wantingToBuild.empty());
2713 174 : assert(!keepGoing || children.empty());
2714 173 : }
2715 :
2716 :
2717 2945 : void Worker::waitForInput()
2718 : {
2719 2945 : printMsg(lvlVomit, "waiting for children");
2720 :
2721 : /* Process output from the file descriptors attached to the
2722 : children, namely log output and output path creation commands.
2723 : We also use this to detect child termination: if we get EOF on
2724 : the logger pipe of a build, we assume that the builder has
2725 : terminated. */
2726 :
2727 : /* If we're monitoring for silence on stdout/stderr, sleep until
2728 : the first deadline for any child. */
2729 : struct timeval timeout;
2730 2945 : if (maxSilentTime != 0) {
2731 0 : time_t oldest = 0;
2732 0 : for (Children::iterator i = children.begin();
2733 : i != children.end(); ++i)
2734 : {
2735 : oldest = oldest == 0 || i->second.lastOutput < oldest
2736 0 : ? i->second.lastOutput : oldest;
2737 : }
2738 0 : time_t now = time(0);
2739 : timeout.tv_sec = (time_t) (oldest + maxSilentTime) <= now ? 0 :
2740 0 : oldest + maxSilentTime - now;
2741 0 : timeout.tv_usec = 0;
2742 0 : printMsg(lvlVomit, format("sleeping %1% seconds") % timeout.tv_sec);
2743 : }
2744 :
2745 : /* Use select() to wait for the input side of any logger pipe to
2746 : become `available'. Note that `available' (i.e., non-blocking)
2747 : includes EOF. */
2748 : fd_set fds;
2749 2945 : FD_ZERO(&fds);
2750 2945 : int fdMax = 0;
2751 5957 : for (Children::iterator i = children.begin();
2752 : i != children.end(); ++i)
2753 : {
2754 6024 : for (set<int>::iterator j = i->second.fds.begin();
2755 : j != i->second.fds.end(); ++j)
2756 : {
2757 3012 : FD_SET(*j, &fds);
2758 3012 : if (*j >= fdMax) fdMax = *j + 1;
2759 : }
2760 : }
2761 :
2762 2945 : if (select(fdMax, &fds, 0, 0, maxSilentTime != 0 ? &timeout : 0) == -1) {
2763 0 : if (errno == EINTR) return;
2764 0 : throw SysError("waiting for input");
2765 : }
2766 :
2767 2945 : time_t now = time(0);
2768 :
2769 : /* Process all available file descriptors. */
2770 :
2771 : /* Since goals may be canceled from inside the loop below (causing
2772 : them go be erased from the `children' map), we have to be
2773 : careful that we don't keep iterators alive across calls to
2774 : cancel(). */
2775 2945 : set<pid_t> pids;
2776 5957 : for (Children::iterator i = children.begin();
2777 : i != children.end(); ++i)
2778 3012 : pids.insert(i->first);
2779 :
2780 5957 : for (set<pid_t>::iterator i = pids.begin();
2781 : i != pids.end(); ++i)
2782 : {
2783 3012 : checkInterrupt();
2784 3012 : Children::iterator j = children.find(*i);
2785 3012 : if (j == children.end()) continue; // child destroyed
2786 3012 : GoalPtr goal = j->second.goal.lock();
2787 3012 : assert(goal);
2788 :
2789 3012 : set<int> fds2(j->second.fds);
2790 6024 : for (set<int>::iterator k = fds2.begin(); k != fds2.end(); ++k) {
2791 3012 : if (FD_ISSET(*k, &fds)) {
2792 : unsigned char buffer[4096];
2793 2947 : ssize_t rd = read(*k, buffer, sizeof(buffer));
2794 2947 : if (rd == -1) {
2795 0 : if (errno != EINTR)
2796 : throw SysError(format("reading from %1%")
2797 0 : % goal->getName());
2798 2947 : } else if (rd == 0) {
2799 130 : debug(format("%1%: got EOF") % goal->getName());
2800 130 : goal->handleEOF(*k);
2801 130 : j->second.fds.erase(*k);
2802 : } else {
2803 2817 : printMsg(lvlVomit, format("%1%: read %2% bytes")
2804 : % goal->getName() % rd);
2805 2817 : string data((char *) buffer, rd);
2806 2817 : goal->handleChildOutput(*k, data);
2807 2817 : j->second.lastOutput = now;
2808 : }
2809 : }
2810 : }
2811 :
2812 3012 : if (maxSilentTime != 0 &&
2813 : now - j->second.lastOutput >= (time_t) maxSilentTime)
2814 : {
2815 0 : printMsg(lvlError,
2816 : format("%1% timed out after %2% seconds of silence")
2817 : % goal->getName() % maxSilentTime);
2818 0 : goal->cancel();
2819 : }
2820 2945 : }
2821 : }
2822 :
2823 :
2824 :
2825 : //////////////////////////////////////////////////////////////////////
2826 :
2827 :
2828 172 : void LocalStore::buildDerivations(const PathSet & drvPaths)
2829 : {
2830 172 : startNest(nest, lvlDebug,
2831 : format("building %1%") % showPaths(drvPaths));
2832 :
2833 172 : Worker worker(*this);
2834 :
2835 172 : Goals goals;
2836 338 : for (PathSet::const_iterator i = drvPaths.begin();
2837 : i != drvPaths.end(); ++i)
2838 166 : goals.insert(worker.makeDerivationGoal(*i));
2839 :
2840 172 : worker.run(goals);
2841 :
2842 171 : PathSet failed;
2843 336 : for (Goals::iterator i = goals.begin(); i != goals.end(); ++i)
2844 165 : if ((*i)->getExitCode() == Goal::ecFailed) {
2845 6 : DerivationGoal * i2 = dynamic_cast<DerivationGoal *>(i->get());
2846 6 : assert(i2);
2847 6 : failed.insert(i2->getDrvPath());
2848 : }
2849 :
2850 171 : if (!failed.empty())
2851 13 : throw Error(format("build of %1% failed") % showPaths(failed));
2852 165 : }
2853 :
2854 :
2855 483 : void LocalStore::ensurePath(const Path & path)
2856 : {
2857 : /* If the path is already valid, we're done. */
2858 483 : if (isValidPath(path)) return;
2859 :
2860 2 : Worker worker(*this);
2861 2 : GoalPtr goal = worker.makeSubstitutionGoal(path);
2862 2 : Goals goals = singleton<Goals>(goal);
2863 :
2864 2 : worker.run(goals);
2865 :
2866 2 : if (goal->getExitCode() != Goal::ecSuccess)
2867 0 : throw Error(format("path `%1%' does not exist and cannot be created") % path);
2868 : }
2869 :
2870 0 :
2871 1106 : }
|
