diff --git a/Godeps/Godeps.json b/Godeps/Godeps.json index ab05934b9..613f484e2 100644 --- a/Godeps/Godeps.json +++ b/Godeps/Godeps.json @@ -38,6 +38,10 @@ "Comment": "null-15", "Rev": "12e4b4183793ac4b061921e7980845e750679fd0" }, + { + "ImportPath": "github.com/camlistore/lock", + "Rev": "ae27720f340952636b826119b58130b9c1a847a0" + }, { "ImportPath": "github.com/gonuts/flag", "Rev": "741a6cbd37a30dedc93f817e7de6aaf0ca38a493" @@ -68,6 +72,10 @@ "Comment": "0.1.0-5-g1976046", "Rev": "1976046c2b0db0b668791b3e541d76a38b7c1af7" }, + { + "ImportPath": "github.com/op/go-logging", + "Rev": "3df864a88c7f005e676db4f026a4fe2f14929be3" + }, { "ImportPath": "github.com/syndtr/goleveldb/leveldb", "Rev": "99056d50e56252fbe0021d5c893defca5a76baf8" diff --git a/Godeps/_workspace/src/github.com/camlistore/lock/.gitignore b/Godeps/_workspace/src/github.com/camlistore/lock/.gitignore new file mode 100644 index 000000000..b25c15b81 --- /dev/null +++ b/Godeps/_workspace/src/github.com/camlistore/lock/.gitignore @@ -0,0 +1 @@ +*~ diff --git a/Godeps/_workspace/src/github.com/camlistore/lock/COPYING b/Godeps/_workspace/src/github.com/camlistore/lock/COPYING new file mode 100644 index 000000000..d64569567 --- /dev/null +++ b/Godeps/_workspace/src/github.com/camlistore/lock/COPYING @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/Godeps/_workspace/src/github.com/camlistore/lock/README.txt b/Godeps/_workspace/src/github.com/camlistore/lock/README.txt new file mode 100644 index 000000000..a9eeb33de --- /dev/null +++ b/Godeps/_workspace/src/github.com/camlistore/lock/README.txt @@ -0,0 +1,3 @@ +File locking library. + +See http://godoc.org/github.com/camlistore/lock diff --git a/Godeps/_workspace/src/github.com/camlistore/lock/lock.go b/Godeps/_workspace/src/github.com/camlistore/lock/lock.go new file mode 100644 index 000000000..6268527b0 --- /dev/null +++ b/Godeps/_workspace/src/github.com/camlistore/lock/lock.go @@ -0,0 +1,158 @@ +/* +Copyright 2013 The Go Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package lock + +import ( + "encoding/json" + "fmt" + "io" + "os" + "path/filepath" + "sync" +) + +// Lock locks the given file, creating the file if necessary. If the +// file already exists, it must have zero size or an error is returned. +// The lock is an exclusive lock (a write lock), but locked files +// should neither be read from nor written to. Such files should have +// zero size and only exist to co-ordinate ownership across processes. +// +// A nil Closer is returned if an error occurred. Otherwise, close that +// Closer to release the lock. +// +// On Linux, FreeBSD and OSX, a lock has the same semantics as fcntl(2)'s +// advisory locks. In particular, closing any other file descriptor for the +// same file will release the lock prematurely. +// +// Attempting to lock a file that is already locked by the current process +// has undefined behavior. +// +// On other operating systems, lock will fallback to using the presence and +// content of a file named name + '.lock' to implement locking behavior. +func Lock(name string) (io.Closer, error) { + return lockFn(name) +} + +var lockFn = lockPortable + +// Portable version not using fcntl. Doesn't handle crashes as gracefully, +// since it can leave stale lock files. +// TODO: write pid of owner to lock file and on race see if pid is +// still alive? +func lockPortable(name string) (io.Closer, error) { + absName, err := filepath.Abs(name) + if err != nil { + return nil, fmt.Errorf("can't Lock file %q: can't find abs path: %v", name, err) + } + fi, err := os.Stat(absName) + if err == nil && fi.Size() > 0 { + if isStaleLock(absName) { + os.Remove(absName) + } else { + return nil, fmt.Errorf("can't Lock file %q: has non-zero size", name) + } + } + f, err := os.OpenFile(absName, os.O_RDWR|os.O_CREATE|os.O_TRUNC|os.O_EXCL, 0666) + if err != nil { + return nil, fmt.Errorf("failed to create lock file %s %v", absName, err) + } + if err := json.NewEncoder(f).Encode(&pidLockMeta{OwnerPID: os.Getpid()}); err != nil { + return nil, err + } + return &lockCloser{f: f, abs: absName}, nil +} + +type pidLockMeta struct { + OwnerPID int +} + +func isStaleLock(path string) bool { + f, err := os.Open(path) + if err != nil { + return false + } + defer f.Close() + var meta pidLockMeta + if json.NewDecoder(f).Decode(&meta) != nil { + return false + } + if meta.OwnerPID == 0 { + return false + } + p, err := os.FindProcess(meta.OwnerPID) + if err != nil { + // e.g. on Windows + return true + } + // On unix, os.FindProcess always is true, so we have to send + // it a signal to see if it's alive. + if signalZero != nil { + if p.Signal(signalZero) != nil { + return true + } + } + return false +} + +var signalZero os.Signal // nil or set by lock_sigzero.go + +type lockCloser struct { + f *os.File + abs string + once sync.Once + err error +} + +func (lc *lockCloser) Close() error { + lc.once.Do(lc.close) + return lc.err +} + +func (lc *lockCloser) close() { + if err := lc.f.Close(); err != nil { + lc.err = err + } + if err := os.Remove(lc.abs); err != nil { + lc.err = err + } +} + +var ( + lockmu sync.Mutex + locked = map[string]bool{} // abs path -> true +) + +// unlocker is used by the darwin and linux implementations with fcntl +// advisory locks. +type unlocker struct { + f *os.File + abs string +} + +func (u *unlocker) Close() error { + lockmu.Lock() + // Remove is not necessary but it's nice for us to clean up. + // If we do do this, though, it needs to be before the + // u.f.Close below. + os.Remove(u.abs) + if err := u.f.Close(); err != nil { + return err + } + delete(locked, u.abs) + lockmu.Unlock() + return nil +} diff --git a/Godeps/_workspace/src/github.com/camlistore/lock/lock_appengine.go b/Godeps/_workspace/src/github.com/camlistore/lock/lock_appengine.go new file mode 100644 index 000000000..ab4cad6ab --- /dev/null +++ b/Godeps/_workspace/src/github.com/camlistore/lock/lock_appengine.go @@ -0,0 +1,32 @@ +// +build appengine + +/* +Copyright 2013 The Go Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package lock + +import ( + "errors" + "io" +) + +func init() { + lockFn = lockAppEngine +} + +func lockAppEngine(name string) (io.Closer, error) { + return nil, errors.New("Lock not available on App Engine") +} diff --git a/Godeps/_workspace/src/github.com/camlistore/lock/lock_darwin_amd64.go b/Godeps/_workspace/src/github.com/camlistore/lock/lock_darwin_amd64.go new file mode 100644 index 000000000..9fea51fe8 --- /dev/null +++ b/Godeps/_workspace/src/github.com/camlistore/lock/lock_darwin_amd64.go @@ -0,0 +1,80 @@ +// +build darwin,amd64 +// +build !appengine + +/* +Copyright 2013 The Go Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package lock + +import ( + "fmt" + "io" + "os" + "path/filepath" + "syscall" + "unsafe" +) + +func init() { + lockFn = lockFcntl +} + +func lockFcntl(name string) (io.Closer, error) { + abs, err := filepath.Abs(name) + if err != nil { + return nil, err + } + lockmu.Lock() + if locked[abs] { + lockmu.Unlock() + return nil, fmt.Errorf("file %q already locked", abs) + } + locked[abs] = true + lockmu.Unlock() + + fi, err := os.Stat(name) + if err == nil && fi.Size() > 0 { + return nil, fmt.Errorf("can't Lock file %q: has non-zero size", name) + } + + f, err := os.Create(name) + if err != nil { + return nil, fmt.Errorf("Lock Create of %s (abs: %s) failed: %v", name, abs, err) + } + + // This type matches C's "struct flock" defined in /usr/include/sys/fcntl.h. + // TODO: move this into the standard syscall package. + k := struct { + Start uint64 // sizeof(off_t): 8 + Len uint64 // sizeof(off_t): 8 + Pid uint32 // sizeof(pid_t): 4 + Type uint16 // sizeof(short): 2 + Whence uint16 // sizeof(short): 2 + }{ + Type: syscall.F_WRLCK, + Whence: uint16(os.SEEK_SET), + Start: 0, + Len: 0, // 0 means to lock the entire file. + Pid: uint32(os.Getpid()), + } + + _, _, errno := syscall.Syscall(syscall.SYS_FCNTL, f.Fd(), uintptr(syscall.F_SETLK), uintptr(unsafe.Pointer(&k))) + if errno != 0 { + f.Close() + return nil, errno + } + return &unlocker{f, abs}, nil +} diff --git a/Godeps/_workspace/src/github.com/camlistore/lock/lock_freebsd.go b/Godeps/_workspace/src/github.com/camlistore/lock/lock_freebsd.go new file mode 100644 index 000000000..d3835d624 --- /dev/null +++ b/Godeps/_workspace/src/github.com/camlistore/lock/lock_freebsd.go @@ -0,0 +1,79 @@ +/* +Copyright 2013 The Go Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package lock + +import ( + "fmt" + "io" + "os" + "path/filepath" + "syscall" + "unsafe" +) + +func init() { + lockFn = lockFcntl +} + +func lockFcntl(name string) (io.Closer, error) { + abs, err := filepath.Abs(name) + if err != nil { + return nil, err + } + lockmu.Lock() + if locked[abs] { + lockmu.Unlock() + return nil, fmt.Errorf("file %q already locked", abs) + } + locked[abs] = true + lockmu.Unlock() + + fi, err := os.Stat(name) + if err == nil && fi.Size() > 0 { + return nil, fmt.Errorf("can't Lock file %q: has non-zero size", name) + } + + f, err := os.Create(name) + if err != nil { + return nil, err + } + + // This type matches C's "struct flock" defined in /usr/include/fcntl.h. + // TODO: move this into the standard syscall package. + k := struct { + Start int64 /* off_t starting offset */ + Len int64 /* off_t len = 0 means until end of file */ + Pid int32 /* pid_t lock owner */ + Type int16 /* short lock type: read/write, etc. */ + Whence int16 /* short type of l_start */ + Sysid int32 /* int remote system id or zero for local */ + }{ + Start: 0, + Len: 0, // 0 means to lock the entire file. + Pid: int32(os.Getpid()), + Type: syscall.F_WRLCK, + Whence: int16(os.SEEK_SET), + Sysid: 0, + } + + _, _, errno := syscall.Syscall(syscall.SYS_FCNTL, f.Fd(), uintptr(syscall.F_SETLK), uintptr(unsafe.Pointer(&k))) + if errno != 0 { + f.Close() + return nil, errno + } + return &unlocker{f, abs}, nil +} diff --git a/Godeps/_workspace/src/github.com/camlistore/lock/lock_linux_amd64.go b/Godeps/_workspace/src/github.com/camlistore/lock/lock_linux_amd64.go new file mode 100644 index 000000000..3a7eb00a6 --- /dev/null +++ b/Godeps/_workspace/src/github.com/camlistore/lock/lock_linux_amd64.go @@ -0,0 +1,80 @@ +// +build linux,amd64 +// +build !appengine + +/* +Copyright 2013 The Go Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package lock + +import ( + "fmt" + "io" + "os" + "path/filepath" + "syscall" + "unsafe" +) + +func init() { + lockFn = lockFcntl +} + +func lockFcntl(name string) (io.Closer, error) { + abs, err := filepath.Abs(name) + if err != nil { + return nil, err + } + lockmu.Lock() + if locked[abs] { + lockmu.Unlock() + return nil, fmt.Errorf("file %q already locked", abs) + } + locked[abs] = true + lockmu.Unlock() + + fi, err := os.Stat(name) + if err == nil && fi.Size() > 0 { + return nil, fmt.Errorf("can't Lock file %q: has non-zero size", name) + } + + f, err := os.Create(name) + if err != nil { + return nil, err + } + + // This type matches C's "struct flock" defined in /usr/include/bits/fcntl.h. + // TODO: move this into the standard syscall package. + k := struct { + Type uint32 + Whence uint32 + Start uint64 + Len uint64 + Pid uint32 + }{ + Type: syscall.F_WRLCK, + Whence: uint32(os.SEEK_SET), + Start: 0, + Len: 0, // 0 means to lock the entire file. + Pid: uint32(os.Getpid()), + } + + _, _, errno := syscall.Syscall(syscall.SYS_FCNTL, f.Fd(), uintptr(syscall.F_SETLK), uintptr(unsafe.Pointer(&k))) + if errno != 0 { + f.Close() + return nil, errno + } + return &unlocker{f, abs}, nil +} diff --git a/Godeps/_workspace/src/github.com/camlistore/lock/lock_linux_arm.go b/Godeps/_workspace/src/github.com/camlistore/lock/lock_linux_arm.go new file mode 100644 index 000000000..c2a0a102e --- /dev/null +++ b/Godeps/_workspace/src/github.com/camlistore/lock/lock_linux_arm.go @@ -0,0 +1,81 @@ +// +build linux,arm +// +build !appengine + +/* +Copyright 2013 The Go Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package lock + +import ( + "fmt" + "io" + "os" + "path/filepath" + "syscall" + "unsafe" +) + +func init() { + lockFn = lockFcntl +} + +func lockFcntl(name string) (io.Closer, error) { + abs, err := filepath.Abs(name) + if err != nil { + return nil, err + } + lockmu.Lock() + if locked[abs] { + lockmu.Unlock() + return nil, fmt.Errorf("file %q already locked", abs) + } + locked[abs] = true + lockmu.Unlock() + + fi, err := os.Stat(name) + if err == nil && fi.Size() > 0 { + return nil, fmt.Errorf("can't Lock file %q: has non-zero size", name) + } + + f, err := os.Create(name) + if err != nil { + return nil, err + } + + // This type matches C's "struct flock" defined in /usr/include/bits/fcntl.h. + // TODO: move this into the standard syscall package. + k := struct { + Type uint16 + Whence uint16 + Start uint32 + Len uint32 + Pid uint32 + }{ + Type: syscall.F_WRLCK, + Whence: uint16(os.SEEK_SET), + Start: 0, + Len: 0, // 0 means to lock the entire file. + Pid: uint32(os.Getpid()), + } + + const F_SETLK = 6 // actual value. syscall package is wrong: golang.org/issue/7059 + _, _, errno := syscall.Syscall(syscall.SYS_FCNTL, f.Fd(), uintptr(F_SETLK), uintptr(unsafe.Pointer(&k))) + if errno != 0 { + f.Close() + return nil, errno + } + return &unlocker{f, abs}, nil +} diff --git a/Godeps/_workspace/src/github.com/camlistore/lock/lock_plan9.go b/Godeps/_workspace/src/github.com/camlistore/lock/lock_plan9.go new file mode 100644 index 000000000..bdf4e2292 --- /dev/null +++ b/Godeps/_workspace/src/github.com/camlistore/lock/lock_plan9.go @@ -0,0 +1,55 @@ +/* +Copyright 2013 The Go Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package lock + +import ( + "fmt" + "io" + "os" + "path/filepath" +) + +func init() { + lockFn = lockPlan9 +} + +func lockPlan9(name string) (io.Closer, error) { + var f *os.File + abs, err := filepath.Abs(name) + if err != nil { + return nil, err + } + lockmu.Lock() + if locked[abs] { + lockmu.Unlock() + return nil, fmt.Errorf("file %q already locked", abs) + } + locked[abs] = true + lockmu.Unlock() + + fi, err := os.Stat(name) + if err == nil && fi.Size() > 0 { + return nil, fmt.Errorf("can't Lock file %q: has non-zero size", name) + } + + f, err = os.OpenFile(name, os.O_RDWR|os.O_CREATE, os.ModeExclusive|0644) + if err != nil { + return nil, fmt.Errorf("Lock Create of %s (abs: %s) failed: %v", name, abs, err) + } + + return &unlocker{f, abs}, nil +} diff --git a/Godeps/_workspace/src/github.com/camlistore/lock/lock_sigzero.go b/Godeps/_workspace/src/github.com/camlistore/lock/lock_sigzero.go new file mode 100644 index 000000000..fd3ba2db1 --- /dev/null +++ b/Godeps/_workspace/src/github.com/camlistore/lock/lock_sigzero.go @@ -0,0 +1,26 @@ +// +build !appengine +// +build linux darwin freebsd openbsd netbsd dragonfly + +/* +Copyright 2013 The Go Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package lock + +import "syscall" + +func init() { + signalZero = syscall.Signal(0) +} diff --git a/Godeps/_workspace/src/github.com/camlistore/lock/lock_test.go b/Godeps/_workspace/src/github.com/camlistore/lock/lock_test.go new file mode 100644 index 000000000..518d2f025 --- /dev/null +++ b/Godeps/_workspace/src/github.com/camlistore/lock/lock_test.go @@ -0,0 +1,131 @@ +/* +Copyright 2013 The Go Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package lock + +import ( + "fmt" + "io/ioutil" + "log" + "os" + "os/exec" + "path/filepath" + "strconv" + "testing" +) + +func TestLock(t *testing.T) { + testLock(t, false) +} + +func TestLockPortable(t *testing.T) { + testLock(t, true) +} + +func TestLockInChild(t *testing.T) { + f := os.Getenv("TEST_LOCK_FILE") + if f == "" { + // not child + return + } + lock := Lock + if v, _ := strconv.ParseBool(os.Getenv("TEST_LOCK_PORTABLE")); v { + lock = lockPortable + } + + lk, err := lock(f) + if err != nil { + log.Fatalf("Lock failed: %v", err) + } + + if v, _ := strconv.ParseBool(os.Getenv("TEST_LOCK_CRASH")); v { + // Simulate a crash, or at least not unlocking the + // lock. We still exit 0 just to simplify the parent + // process exec code. + os.Exit(0) + } + lk.Close() +} + +func testLock(t *testing.T, portable bool) { + lock := Lock + if portable { + lock = lockPortable + } + + td, err := ioutil.TempDir("", "") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(td) + + path := filepath.Join(td, "foo.lock") + + childLock := func(crash bool) error { + cmd := exec.Command(os.Args[0], "-test.run=LockInChild$") + cmd.Env = []string{"TEST_LOCK_FILE=" + path} + if portable { + cmd.Env = append(cmd.Env, "TEST_LOCK_PORTABLE=1") + } + if crash { + cmd.Env = append(cmd.Env, "TEST_LOCK_CRASH=1") + } + out, err := cmd.CombinedOutput() + t.Logf("Child output: %q (err %v)", out, err) + if err != nil { + return fmt.Errorf("Child Process lock of %s failed: %v %s", path, err, out) + } + return nil + } + + t.Logf("Locking in crashing child...") + if err := childLock(true); err != nil { + t.Fatalf("first lock in child process: %v", err) + } + + t.Logf("Locking+unlocking in child...") + if err := childLock(false); err != nil { + t.Fatalf("lock in child process after crashing child: %v", err) + } + + t.Logf("Locking in parent...") + lk1, err := lock(path) + if err != nil { + t.Fatal(err) + } + + t.Logf("Again in parent...") + _, err = lock(path) + if err == nil { + t.Fatal("expected second lock to fail") + } + + t.Logf("Locking in child...") + if childLock(false) == nil { + t.Fatalf("expected lock in child process to fail") + } + + t.Logf("Unlocking lock in parent") + if err := lk1.Close(); err != nil { + t.Fatal(err) + } + + lk3, err := lock(path) + if err != nil { + t.Fatal(err) + } + lk3.Close() +} diff --git a/Godeps/_workspace/src/github.com/op/go-logging/.travis.yml b/Godeps/_workspace/src/github.com/op/go-logging/.travis.yml new file mode 100644 index 000000000..70e012b81 --- /dev/null +++ b/Godeps/_workspace/src/github.com/op/go-logging/.travis.yml @@ -0,0 +1,6 @@ +language: go + +go: + - 1.0 + - 1.1 + - tip diff --git a/Godeps/_workspace/src/github.com/op/go-logging/CONTRIBUTORS b/Godeps/_workspace/src/github.com/op/go-logging/CONTRIBUTORS new file mode 100644 index 000000000..958416ef1 --- /dev/null +++ b/Godeps/_workspace/src/github.com/op/go-logging/CONTRIBUTORS @@ -0,0 +1,5 @@ +Alec Thomas +Guilhem Lettron +Ivan Daniluk +Nimi Wariboko Jr +Róbert Selvek diff --git a/Godeps/_workspace/src/github.com/op/go-logging/LICENSE b/Godeps/_workspace/src/github.com/op/go-logging/LICENSE new file mode 100644 index 000000000..f1f6cfcef --- /dev/null +++ b/Godeps/_workspace/src/github.com/op/go-logging/LICENSE @@ -0,0 +1,27 @@ +Copyright (c) 2013 Örjan Persson. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/Godeps/_workspace/src/github.com/op/go-logging/README.md b/Godeps/_workspace/src/github.com/op/go-logging/README.md new file mode 100644 index 000000000..81d1705fc --- /dev/null +++ b/Godeps/_workspace/src/github.com/op/go-logging/README.md @@ -0,0 +1,83 @@ +## Golang logging library + +[![Build Status](https://travis-ci.org/op/go-logging.png)](https://travis-ci.org/op/go-logging) + +Package logging implements a logging infrastructure for Go. Its output format +is customizable and supports different logging backends like syslog, file and +memory. Multiple backends can be utilized with different log levels per backend +and logger. + +## Example + +Let's have a look at an [example](examples/example.go) which demonstrates most +of the features found in this library. + +[![Example Output](examples/example.png)](examples/example.go) + +```go +package main + +import ( + "os" + + "github.com/op/go-logging" +) + +var log = logging.MustGetLogger("example") + +// Example format string. Everything except the message has a custom color +// which is dependent on the log level. Many fields have a custom output +// formatting too, eg. the time returns the hour down to the milli second. +var format = "%{color}%{time:15:04:05.000000} ▶ %{level:.4s} %{id:03x}%{color:reset} %{message}" + +// Password is just an example type implementing the Redactor interface. Any +// time this is logged, the Redacted() function will be called. +type Password string + +func (p Password) Redacted() interface{} { + return logging.Redact(string(p)) +} + +func main() { + // Setup one stderr and one syslog backend and combine them both into one + // logging backend. By default stderr is used with the standard log flag. + logBackend := logging.NewLogBackend(os.Stderr, "", 0) + syslogBackend, err := logging.NewSyslogBackend("") + if err != nil { + log.Fatal(err) + } + logging.SetBackend(logBackend, syslogBackend) + logging.SetFormatter(logging.MustStringFormatter(format)) + + // For "example", set the log level to DEBUG and ERROR. + for _, level := range []logging.Level{logging.DEBUG, logging.ERROR} { + logging.SetLevel(level, "example") + + log.Debug("debug %s", Password("secret")) + log.Info("info") + log.Notice("notice") + log.Warning("warning") + log.Error("err") + log.Critical("crit") + } +} +``` + +## Installing + +### Using *go get* + + $ go get github.com/op/go-logging + +After this command *go-logging* is ready to use. Its source will be in: + + $GOROOT/src/pkg/github.com/op/go-logging + +You can use `go get -u` to update the package. + +## Documentation + +For docs, see http://godoc.org/github.com/op/go-logging or run: + + $ godoc github.com/op/go-logging + diff --git a/Godeps/_workspace/src/github.com/op/go-logging/backend.go b/Godeps/_workspace/src/github.com/op/go-logging/backend.go new file mode 100644 index 000000000..6cd589cab --- /dev/null +++ b/Godeps/_workspace/src/github.com/op/go-logging/backend.go @@ -0,0 +1,39 @@ +// Copyright 2013, Örjan Persson. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package logging + +// defaultBackend is the backend used for all logging calls. +var defaultBackend LeveledBackend + +// Backend is the interface which a log backend need to implement to be able to +// be used as a logging backend. +type Backend interface { + Log(Level, int, *Record) error +} + +// Set backend replaces the backend currently set with the given new logging +// backend. +func SetBackend(backends ...Backend) LeveledBackend { + var backend Backend + if len(backends) == 1 { + backend = backends[0] + } else { + backend = MultiLogger(backends...) + } + + defaultBackend = AddModuleLevel(backend) + return defaultBackend +} + +// SetLevel sets the logging level for the specified module. The module +// corresponds to the string specified in GetLogger. +func SetLevel(level Level, module string) { + defaultBackend.SetLevel(level, module) +} + +// GetLevel returns the logging level for the specified module. +func GetLevel(module string) Level { + return defaultBackend.GetLevel(module) +} diff --git a/Godeps/_workspace/src/github.com/op/go-logging/examples/example.go b/Godeps/_workspace/src/github.com/op/go-logging/examples/example.go new file mode 100644 index 000000000..58287ad0a --- /dev/null +++ b/Godeps/_workspace/src/github.com/op/go-logging/examples/example.go @@ -0,0 +1,46 @@ +package main + +import ( + "os" + + "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/op/go-logging" +) + +var log = logging.MustGetLogger("example") + +// Example format string. Everything except the message has a custom color +// which is dependent on the log level. Many fields have a custom output +// formatting too, eg. the time returns the hour down to the milli second. +var format = "%{color}%{time:15:04:05.000000} %{shortfunc} ▶ %{level:.4s} %{id:03x}%{color:reset} %{message}" + +// Password is just an example type implementing the Redactor interface. Any +// time this is logged, the Redacted() function will be called. +type Password string + +func (p Password) Redacted() interface{} { + return logging.Redact(string(p)) +} + +func main() { + // Setup one stderr and one syslog backend and combine them both into one + // logging backend. By default stderr is used with the standard log flag. + logBackend := logging.NewLogBackend(os.Stderr, "", 0) + syslogBackend, err := logging.NewSyslogBackend("") + if err != nil { + log.Fatal(err) + } + logging.SetBackend(logBackend, syslogBackend) + logging.SetFormatter(logging.MustStringFormatter(format)) + + // For "example", set the log level to DEBUG and ERROR. + for _, level := range []logging.Level{logging.DEBUG, logging.ERROR} { + logging.SetLevel(level, "example") + + log.Debug("debug %s", Password("secret")) + log.Info("info") + log.Notice("notice") + log.Warning("warning") + log.Error("err") + log.Critical("crit") + } +} diff --git a/Godeps/_workspace/src/github.com/op/go-logging/examples/example.png b/Godeps/_workspace/src/github.com/op/go-logging/examples/example.png new file mode 100644 index 000000000..cf42503db Binary files /dev/null and b/Godeps/_workspace/src/github.com/op/go-logging/examples/example.png differ diff --git a/Godeps/_workspace/src/github.com/op/go-logging/format.go b/Godeps/_workspace/src/github.com/op/go-logging/format.go new file mode 100644 index 000000000..af9e98750 --- /dev/null +++ b/Godeps/_workspace/src/github.com/op/go-logging/format.go @@ -0,0 +1,349 @@ +// Copyright 2013, Örjan Persson. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package logging + +import ( + "bytes" + "errors" + "fmt" + "io" + "os" + "path" + "path/filepath" + "regexp" + "runtime" + "strings" + "sync" + "time" +) + +// TODO see Formatter interface in fmt/print.go +// TODO try text/template, maybe it have enough performance +// TODO other template systems? +// TODO make it possible to specify formats per backend? +type fmtVerb int + +const ( + fmtVerbTime fmtVerb = iota + fmtVerbLevel + fmtVerbId + fmtVerbPid + fmtVerbProgram + fmtVerbModule + fmtVerbMessage + fmtVerbLongfile + fmtVerbShortfile + fmtVerbLongpkg + fmtVerbShortpkg + fmtVerbLongfunc + fmtVerbShortfunc + fmtVerbLevelColor + + // Keep last, there are no match for these below. + fmtVerbUnknown + fmtVerbStatic +) + +var fmtVerbs = []string{ + "time", + "level", + "id", + "pid", + "program", + "module", + "message", + "longfile", + "shortfile", + "longpkg", + "shortpkg", + "longfunc", + "shortfunc", + "color", +} + +const rfc3339Milli = "2006-01-02T15:04:05.999Z07:00" + +var defaultVerbsLayout = []string{ + rfc3339Milli, + "s", + "d", + "d", + "s", + "s", + "s", + "s", + "s", + "s", + "s", + "s", + "s", + "", +} + +var ( + pid = os.Getpid() + program = filepath.Base(os.Args[0]) +) + +func getFmtVerbByName(name string) fmtVerb { + for i, verb := range fmtVerbs { + if name == verb { + return fmtVerb(i) + } + } + return fmtVerbUnknown +} + +// Formatter is the required interface for a custom log record formatter. +type Formatter interface { + Format(calldepth int, r *Record, w io.Writer) error +} + +// formatter is used by all backends unless otherwise overriden. +var formatter struct { + sync.RWMutex + def Formatter +} + +func getFormatter() Formatter { + formatter.RLock() + defer formatter.RUnlock() + return formatter.def +} + +var ( + // DefaultFormatter is the default formatter used and is only the message. + DefaultFormatter Formatter = MustStringFormatter("%{message}") + + // Glog format + GlogFormatter Formatter = MustStringFormatter("%{level:.1s}%{time:0102 15:04:05.999999} %{pid} %{shortfile}] %{message}") +) + +// SetFormatter sets the default formatter for all new backends. A backend will +// fetch this value once it is needed to format a record. Note that backends +// will cache the formatter after the first point. For now, make sure to set +// the formatter before logging. +func SetFormatter(f Formatter) { + formatter.Lock() + defer formatter.Unlock() + formatter.def = f +} + +var formatRe *regexp.Regexp = regexp.MustCompile(`%{([a-z]+)(?::(.*?[^\\]))?}`) + +type part struct { + verb fmtVerb + layout string +} + +// stringFormatter contains a list of parts which explains how to build the +// formatted string passed on to the logging backend. +type stringFormatter struct { + parts []part +} + +// NewStringFormatter returns a new Formatter which outputs the log record as a +// string based on the 'verbs' specified in the format string. +// +// The verbs: +// +// General: +// %{id} Sequence number for log message (uint64). +// %{pid} Process id (int) +// %{time} Time when log occurred (time.Time) +// %{level} Log level (Level) +// %{module} Module (string) +// %{program} Basename of os.Args[0] (string) +// %{message} Message (string) +// %{longfile} Full file name and line number: /a/b/c/d.go:23 +// %{shortfile} Final file name element and line number: d.go:23 +// %{color} ANSI color based on log level +// +// For normal types, the output can be customized by using the 'verbs' defined +// in the fmt package, eg. '%{id:04d}' to make the id output be '%04d' as the +// format string. +// +// For time.Time, use the same layout as time.Format to change the time format +// when output, eg "2006-01-02T15:04:05.999Z-07:00". +// +// For the 'color' verb, the output can be adjusted to either use bold colors, +// i.e., '%{color:bold}' or to reset the ANSI attributes, i.e., +// '%{color:reset}' Note that if you use the color verb explicitly, be sure to +// reset it or else the color state will persist past your log message. e.g., +// "%{color:bold}%{time:15:04:05} %{level:-8s}%{color:reset} %{message}" will +// just colorize the time and level, leaving the message uncolored. +// +// There's also a couple of experimental 'verbs'. These are exposed to get +// feedback and needs a bit of tinkering. Hence, they might change in the +// future. +// +// Experimental: +// %{longpkg} Full package path, eg. github.com/go-logging +// %{shortpkg} Base package path, eg. go-logging +// %{longfunc} Full function name, eg. littleEndian.PutUint32 +// %{shortfunc} Base function name, eg. PutUint32 +func NewStringFormatter(format string) (*stringFormatter, error) { + var fmter = &stringFormatter{} + + // Find the boundaries of all %{vars} + matches := formatRe.FindAllStringSubmatchIndex(format, -1) + if matches == nil { + return nil, errors.New("logger: invalid log format: " + format) + } + + // Collect all variables and static text for the format + prev := 0 + for _, m := range matches { + start, end := m[0], m[1] + if start > prev { + fmter.add(fmtVerbStatic, format[prev:start]) + } + + name := format[m[2]:m[3]] + verb := getFmtVerbByName(name) + if verb == fmtVerbUnknown { + return nil, errors.New("logger: unknown variable: " + name) + } + + // Handle layout customizations or use the default. If this is not for the + // time or color formatting, we need to prefix with %. + layout := defaultVerbsLayout[verb] + if m[4] != -1 { + layout = format[m[4]:m[5]] + } + if verb != fmtVerbTime && verb != fmtVerbLevelColor { + layout = "%" + layout + } + + fmter.add(verb, layout) + prev = end + } + end := format[prev:] + if end != "" { + fmter.add(fmtVerbStatic, end) + } + + // Make a test run to make sure we can format it correctly. + t, err := time.Parse(time.RFC3339, "2010-02-04T21:00:57-08:00") + if err != nil { + panic(err) + } + r := &Record{ + Id: 12345, + Time: t, + Module: "logger", + fmt: "hello %s", + args: []interface{}{"go"}, + } + if err := fmter.Format(0, r, &bytes.Buffer{}); err != nil { + return nil, err + } + + formatter.def = fmter + + return fmter, nil +} + +// MustStringFormatter is equivalent to NewStringFormatter with a call to panic +// on error. +func MustStringFormatter(format string) *stringFormatter { + f, err := NewStringFormatter(format) + if err != nil { + panic("Failed to initialized string formatter: " + err.Error()) + } + return f +} + +func (f *stringFormatter) add(verb fmtVerb, layout string) { + f.parts = append(f.parts, part{verb, layout}) +} + +func (f *stringFormatter) Format(calldepth int, r *Record, output io.Writer) error { + for _, part := range f.parts { + if part.verb == fmtVerbStatic { + output.Write([]byte(part.layout)) + } else if part.verb == fmtVerbTime { + output.Write([]byte(r.Time.Format(part.layout))) + } else if part.verb == fmtVerbLevelColor { + if part.layout == "bold" { + output.Write([]byte(boldcolors[r.Level])) + } else if part.layout == "reset" { + output.Write([]byte("\033[0m")) + } else { + output.Write([]byte(colors[r.Level])) + } + } else { + var v interface{} + switch part.verb { + case fmtVerbLevel: + v = r.Level + break + case fmtVerbId: + v = r.Id + break + case fmtVerbPid: + v = pid + break + case fmtVerbProgram: + v = program + break + case fmtVerbModule: + v = r.Module + break + case fmtVerbMessage: + v = r.Message() + break + case fmtVerbLongfile, fmtVerbShortfile: + _, file, line, ok := runtime.Caller(calldepth + 1) + if !ok { + file = "???" + line = 0 + } else if part.verb == fmtVerbShortfile { + file = filepath.Base(file) + } + v = fmt.Sprintf("%s:%d", file, line) + case fmtVerbLongfunc, fmtVerbShortfunc, + fmtVerbLongpkg, fmtVerbShortpkg: + // TODO cache pc + v = "???" + if pc, _, _, ok := runtime.Caller(calldepth + 1); ok { + if f := runtime.FuncForPC(pc); f != nil { + v = formatFuncName(part.verb, f.Name()) + } + } + default: + panic("unhandled format part") + } + fmt.Fprintf(output, part.layout, v) + } + } + return nil +} + +// formatFuncName tries to extract certain part of the runtime formatted +// function name to some pre-defined variation. +// +// This function is known to not work properly if the package path or name +// contains a dot. +func formatFuncName(v fmtVerb, f string) string { + i := strings.LastIndex(f, "/") + j := strings.Index(f[i+1:], ".") + if j < 1 { + return "???" + } + pkg, fun := f[:i+j+1], f[i+j+2:] + switch v { + case fmtVerbLongpkg: + return pkg + case fmtVerbShortpkg: + return path.Base(pkg) + case fmtVerbLongfunc: + return fun + case fmtVerbShortfunc: + i = strings.LastIndex(fun, ".") + return fun[i+1:] + } + panic("unexpected func formatter") +} diff --git a/Godeps/_workspace/src/github.com/op/go-logging/format_test.go b/Godeps/_workspace/src/github.com/op/go-logging/format_test.go new file mode 100644 index 000000000..d77db0564 --- /dev/null +++ b/Godeps/_workspace/src/github.com/op/go-logging/format_test.go @@ -0,0 +1,158 @@ +// Copyright 2013, Örjan Persson. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package logging + +import ( + "bytes" + "testing" +) + +func TestFormat(t *testing.T) { + backend := InitForTesting(DEBUG) + + f, err := NewStringFormatter("%{shortfile} %{time:2006-01-02T15:04:05} %{level:.1s} %{id:04d} %{module} %{message}") + if err != nil { + t.Fatalf("failed to set format: %s", err) + } + SetFormatter(f) + + log := MustGetLogger("module") + log.Debug("hello") + + line := MemoryRecordN(backend, 0).Formatted(0) + if "format_test.go:24 1970-01-01T00:00:00 D 0001 module hello" != line { + t.Errorf("Unexpected format: %s", line) + } +} + +func logAndGetLine(backend *MemoryBackend) string { + MustGetLogger("foo").Debug("hello") + return MemoryRecordN(backend, 0).Formatted(1) +} + +func realFunc(backend *MemoryBackend) string { + return logAndGetLine(backend) +} + +type structFunc struct{} + +func (structFunc) Log(backend *MemoryBackend) string { + return logAndGetLine(backend) +} + +func TestRealFuncFormat(t *testing.T) { + backend := InitForTesting(DEBUG) + SetFormatter(MustStringFormatter("%{shortfunc}")) + + line := realFunc(backend) + if "realFunc" != line { + t.Errorf("Unexpected format: %s", line) + } +} + +func TestStructFuncFormat(t *testing.T) { + backend := InitForTesting(DEBUG) + SetFormatter(MustStringFormatter("%{longfunc}")) + + var x structFunc + line := x.Log(backend) + if "structFunc.Log" != line { + t.Errorf("Unexpected format: %s", line) + } +} + +func TestVarFuncFormat(t *testing.T) { + backend := InitForTesting(DEBUG) + SetFormatter(MustStringFormatter("%{shortfunc}")) + + var varFunc = func() string { + return logAndGetLine(backend) + } + + line := varFunc() + if "???" == line || "TestVarFuncFormat" == line || "varFunc" == line { + t.Errorf("Unexpected format: %s", line) + } +} + +func TestFormatFuncName(t *testing.T) { + var tests = []struct { + filename string + longpkg string + shortpkg string + longfunc string + shortfunc string + }{ + {"", + "???", + "???", + "???", + "???"}, + {"main", + "???", + "???", + "???", + "???"}, + {"main.", + "main", + "main", + "", + ""}, + {"main.main", + "main", + "main", + "main", + "main"}, + {"github.com/op/go-logging.func·001", + "github.com/op/go-logging", + "go-logging", + "func·001", + "func·001"}, + {"github.com/op/go-logging.stringFormatter.Format", + "github.com/op/go-logging", + "go-logging", + "stringFormatter.Format", + "Format"}, + } + + var v string + for _, test := range tests { + v = formatFuncName(fmtVerbLongpkg, test.filename) + if test.longpkg != v { + t.Errorf("%s != %s", test.longpkg, v) + } + v = formatFuncName(fmtVerbShortpkg, test.filename) + if test.shortpkg != v { + t.Errorf("%s != %s", test.shortpkg, v) + } + v = formatFuncName(fmtVerbLongfunc, test.filename) + if test.longfunc != v { + t.Errorf("%s != %s", test.longfunc, v) + } + v = formatFuncName(fmtVerbShortfunc, test.filename) + if test.shortfunc != v { + t.Errorf("%s != %s", test.shortfunc, v) + } + } +} + +func BenchmarkStringFormatter(b *testing.B) { + fmt := "%{time:2006-01-02T15:04:05} %{level:.1s} %{id:04d} %{module} %{message}" + f := MustStringFormatter(fmt) + + backend := InitForTesting(DEBUG) + buf := &bytes.Buffer{} + log := MustGetLogger("module") + log.Debug("") + record := MemoryRecordN(backend, 0) + + b.ResetTimer() + for i := 0; i < b.N; i++ { + if err := f.Format(1, record, buf); err != nil { + b.Fatal(err) + buf.Truncate(0) + } + } +} diff --git a/Godeps/_workspace/src/github.com/op/go-logging/level.go b/Godeps/_workspace/src/github.com/op/go-logging/level.go new file mode 100644 index 000000000..ca1f0bf23 --- /dev/null +++ b/Godeps/_workspace/src/github.com/op/go-logging/level.go @@ -0,0 +1,123 @@ +// Copyright 2013, Örjan Persson. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package logging + +import ( + "errors" + "strings" + "sync" +) + +var ErrInvalidLogLevel = errors.New("logger: invalid log level") + +// Level defines all available log levels for log messages. +type Level int + +const ( + CRITICAL Level = iota + ERROR + WARNING + NOTICE + INFO + DEBUG +) + +var levelNames = []string{ + "CRITICAL", + "ERROR", + "WARNING", + "NOTICE", + "INFO", + "DEBUG", +} + +// String returns the string representation of a logging level. +func (p Level) String() string { + return levelNames[p] +} + +// LogLevel returns the log level from a string representation. +func LogLevel(level string) (Level, error) { + for i, name := range levelNames { + if strings.EqualFold(name, level) { + return Level(i), nil + } + } + return ERROR, ErrInvalidLogLevel +} + +type Leveled interface { + GetLevel(string) Level + SetLevel(Level, string) + IsEnabledFor(Level, string) bool +} + +// LeveledBackend is a log backend with additional knobs for setting levels on +// individual modules to different levels. +type LeveledBackend interface { + Backend + Leveled +} + +type moduleLeveled struct { + levels map[string]Level + backend Backend + formatter Formatter + once sync.Once +} + +// AddModuleLevel wraps a log backend with knobs to have different log levels +// for different modules. +func AddModuleLevel(backend Backend) LeveledBackend { + var leveled LeveledBackend + var ok bool + if leveled, ok = backend.(LeveledBackend); !ok { + leveled = &moduleLeveled{ + levels: make(map[string]Level), + backend: backend, + } + } + return leveled +} + +// GetLevel returns the log level for the given module. +func (l *moduleLeveled) GetLevel(module string) Level { + level, exists := l.levels[module] + if exists == false { + level, exists = l.levels[""] + // no configuration exists, default to debug + if exists == false { + level = DEBUG + } + } + return level +} + +// SetLevel sets the log level for the given module. +func (l *moduleLeveled) SetLevel(level Level, module string) { + l.levels[module] = level +} + +// IsEnabledFor will return true if logging is enabled for the given module. +func (l *moduleLeveled) IsEnabledFor(level Level, module string) bool { + return level <= l.GetLevel(module) +} + +func (l *moduleLeveled) Log(level Level, calldepth int, rec *Record) (err error) { + if l.IsEnabledFor(level, rec.Module) { + rec.formatter = l.getFormatterAndCacheCurrent() + err = l.backend.Log(level, calldepth+1, rec) + } + return +} + +func (l *moduleLeveled) getFormatterAndCacheCurrent() Formatter { + l.once.Do(func() { + if l.formatter == nil { + l.formatter = getFormatter() + } + }) + return l.formatter +} diff --git a/Godeps/_workspace/src/github.com/op/go-logging/level_test.go b/Godeps/_workspace/src/github.com/op/go-logging/level_test.go new file mode 100644 index 000000000..c8f9a3733 --- /dev/null +++ b/Godeps/_workspace/src/github.com/op/go-logging/level_test.go @@ -0,0 +1,76 @@ +// Copyright 2013, Örjan Persson. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package logging + +import "testing" + +func TestLevelString(t *testing.T) { + // Make sure all levels can be converted from string -> constant -> string + for _, name := range levelNames { + level, err := LogLevel(name) + if err != nil { + t.Errorf("failed to get level: %v", err) + continue + } + + if level.String() != name { + t.Errorf("invalid level conversion: %v != %v", level, name) + } + } +} + +func TestLevelLogLevel(t *testing.T) { + tests := []struct { + expected Level + level string + }{ + {-1, "bla"}, + {INFO, "iNfO"}, + {ERROR, "error"}, + {WARNING, "warninG"}, + } + + for _, test := range tests { + level, err := LogLevel(test.level) + if err != nil { + if test.expected == -1 { + continue + } else { + t.Errorf("failed to convert %s: %s", test.level, err) + } + } + if test.expected != level { + t.Errorf("failed to convert %s to level: %s != %s", test.level, test.expected, level) + } + } +} + +func TestLevelModuleLevel(t *testing.T) { + backend := NewMemoryBackend(128) + + leveled := AddModuleLevel(backend) + leveled.SetLevel(NOTICE, "") + leveled.SetLevel(ERROR, "foo") + leveled.SetLevel(INFO, "foo.bar") + leveled.SetLevel(WARNING, "bar") + + expected := []struct { + level Level + module string + }{ + {NOTICE, ""}, + {NOTICE, "something"}, + {ERROR, "foo"}, + {INFO, "foo.bar"}, + {WARNING, "bar"}, + } + + for _, e := range expected { + actual := leveled.GetLevel(e.module) + if e.level != actual { + t.Errorf("unexpected level in %s: %s != %s", e.module, e.level, actual) + } + } +} diff --git a/Godeps/_workspace/src/github.com/op/go-logging/log.go b/Godeps/_workspace/src/github.com/op/go-logging/log.go new file mode 100644 index 000000000..f009f8af5 --- /dev/null +++ b/Godeps/_workspace/src/github.com/op/go-logging/log.go @@ -0,0 +1,80 @@ +// Copyright 2013, Örjan Persson. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package logging + +import ( + "bytes" + "fmt" + "io" + "log" +) + +// TODO initialize here +var colors []string +var boldcolors []string + +type color int + +const ( + colorBlack = (iota + 30) + colorRed + colorGreen + colorYellow + colorBlue + colorMagenta + colorCyan + colorWhite +) + +// LogBackend utilizes the standard log module. +type LogBackend struct { + Logger *log.Logger + Color bool +} + +// NewLogBackend creates a new LogBackend. +func NewLogBackend(out io.Writer, prefix string, flag int) *LogBackend { + return &LogBackend{Logger: log.New(out, prefix, flag)} +} + +func (b *LogBackend) Log(level Level, calldepth int, rec *Record) error { + if b.Color { + buf := &bytes.Buffer{} + buf.Write([]byte(colors[level])) + buf.Write([]byte(rec.Formatted(calldepth + 1))) + buf.Write([]byte("\033[0m")) + // For some reason, the Go logger arbitrarily decided "2" was the correct + // call depth... + return b.Logger.Output(calldepth+2, buf.String()) + } else { + return b.Logger.Output(calldepth+2, rec.Formatted(calldepth+1)) + } + panic("should not be reached") +} + +func colorSeq(color color) string { + return fmt.Sprintf("\033[%dm", int(color)) +} + +func colorSeqBold(color color) string { + return fmt.Sprintf("\033[%d;1m", int(color)) +} + +func init() { + colors = []string{ + CRITICAL: colorSeq(colorMagenta), + ERROR: colorSeq(colorRed), + WARNING: colorSeq(colorYellow), + NOTICE: colorSeq(colorGreen), + DEBUG: colorSeq(colorCyan), + } + boldcolors = []string{ + CRITICAL: colorSeqBold(colorMagenta), + ERROR: colorSeqBold(colorRed), + WARNING: colorSeqBold(colorYellow), + NOTICE: colorSeqBold(colorGreen), + DEBUG: colorSeqBold(colorCyan), + } +} diff --git a/Godeps/_workspace/src/github.com/op/go-logging/log_test.go b/Godeps/_workspace/src/github.com/op/go-logging/log_test.go new file mode 100644 index 000000000..62a48ead9 --- /dev/null +++ b/Godeps/_workspace/src/github.com/op/go-logging/log_test.go @@ -0,0 +1,95 @@ +// Copyright 2013, Örjan Persson. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package logging + +import ( + "bytes" + "log" + "strings" + "testing" +) + +func TestLogCalldepth(t *testing.T) { + buf := &bytes.Buffer{} + SetBackend(NewLogBackend(buf, "", log.Lshortfile)) + SetFormatter(MustStringFormatter("%{shortfile} %{level} %{message}")) + + log := MustGetLogger("test") + log.Info("test filename") + + parts := strings.SplitN(buf.String(), " ", 2) + + // Verify that the correct filename is registered by the stdlib logger + if !strings.HasPrefix(parts[0], "log_test.go:") { + t.Errorf("incorrect filename: %s", parts[0]) + } + // Verify that the correct filename is registered by go-logging + if !strings.HasPrefix(parts[1], "log_test.go:") { + t.Errorf("incorrect filename: %s", parts[1]) + } +} + +func BenchmarkLogMemoryBackendIgnored(b *testing.B) { + b.StopTimer() + backend := SetBackend(NewMemoryBackend(1024)) + backend.SetLevel(INFO, "") + RunLogBenchmark(b) +} + +func BenchmarkLogMemoryBackend(b *testing.B) { + b.StopTimer() + backend := SetBackend(NewMemoryBackend(1024)) + backend.SetLevel(DEBUG, "") + RunLogBenchmark(b) +} + +func BenchmarkLogChannelMemoryBackend(b *testing.B) { + b.StopTimer() + channelBackend := NewChannelMemoryBackend(1024) + backend := SetBackend(channelBackend) + backend.SetLevel(DEBUG, "") + RunLogBenchmark(b) + channelBackend.Flush() +} + +func BenchmarkLogLogBackend(b *testing.B) { + b.StopTimer() + backend := SetBackend(NewLogBackend(&bytes.Buffer{}, "", 0)) + backend.SetLevel(DEBUG, "") + RunLogBenchmark(b) +} + +func BenchmarkLogLogBackendColor(b *testing.B) { + b.StopTimer() + colorizer := NewLogBackend(&bytes.Buffer{}, "", 0) + colorizer.Color = true + backend := SetBackend(colorizer) + backend.SetLevel(DEBUG, "") + RunLogBenchmark(b) +} + +func BenchmarkLogLogBackendStdFlags(b *testing.B) { + b.StopTimer() + backend := SetBackend(NewLogBackend(&bytes.Buffer{}, "", log.LstdFlags)) + backend.SetLevel(DEBUG, "") + RunLogBenchmark(b) +} + +func BenchmarkLogLogBackendLongFileFlag(b *testing.B) { + b.StopTimer() + backend := SetBackend(NewLogBackend(&bytes.Buffer{}, "", log.Llongfile)) + backend.SetLevel(DEBUG, "") + RunLogBenchmark(b) +} + +func RunLogBenchmark(b *testing.B) { + password := Password("foo") + log := MustGetLogger("test") + + b.StartTimer() + for i := 0; i < b.N; i++ { + log.Debug("log line for %d and this is rectified: %s", i, password) + } +} diff --git a/Godeps/_workspace/src/github.com/op/go-logging/logger.go b/Godeps/_workspace/src/github.com/op/go-logging/logger.go new file mode 100644 index 000000000..47bbd444b --- /dev/null +++ b/Godeps/_workspace/src/github.com/op/go-logging/logger.go @@ -0,0 +1,213 @@ +// Copyright 2013, Örjan Persson. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package logging implements a logging infrastructure for Go. It supports +// different logging backends like syslog, file and memory. Multiple backends +// can be utilized with different log levels per backend and logger. +package logging + +import ( + "bytes" + "fmt" + "log" + "os" + "strings" + "sync/atomic" + "time" +) + +// Redactor is an interface for types that may contain sensitive information +// (like passwords), which shouldn't be printed to the log. The idea was found +// in relog as part of the vitness project. +type Redactor interface { + Redacted() interface{} +} + +// Redact returns a string of * having the same length as s. +func Redact(s string) string { + return strings.Repeat("*", len(s)) +} + +var ( + // Sequence number is incremented and utilized for all log records created. + sequenceNo uint64 + + // timeNow is a customizable for testing purposes. + timeNow = time.Now +) + +// Record represents a log record and contains the timestamp when the record +// was created, an increasing id, filename and line and finally the actual +// formatted log line. +type Record struct { + Id uint64 + Time time.Time + Module string + Level Level + + // message is kept as a pointer to have shallow copies update this once + // needed. + message *string + args []interface{} + fmt string + formatter Formatter + formatted string +} + +func (r *Record) Formatted(calldepth int) string { + if r.formatted == "" { + var buf bytes.Buffer + r.formatter.Format(calldepth+1, r, &buf) + r.formatted = buf.String() + } + return r.formatted +} + +func (r *Record) Message() string { + if r.message == nil { + // Redact the arguments that implements the Redactor interface + for i, arg := range r.args { + if redactor, ok := arg.(Redactor); ok == true { + r.args[i] = redactor.Redacted() + } + } + msg := fmt.Sprintf(r.fmt, r.args...) + r.message = &msg + } + return *r.message +} + +type Logger struct { + Module string +} + +// TODO call NewLogger and remove MustGetLogger? +// GetLogger creates and returns a Logger object based on the module name. +func GetLogger(module string) (*Logger, error) { + return &Logger{Module: module}, nil +} + +// MustGetLogger is like GetLogger but panics if the logger can't be created. +// It simplifies safe initialization of a global logger for eg. a package. +func MustGetLogger(module string) *Logger { + logger, err := GetLogger(module) + if err != nil { + panic("logger: " + module + ": " + err.Error()) + } + return logger +} + +// Reset restores the internal state of the logging library. +func Reset() { + // TODO make a global Init() method to be less magic? or make it such that + // if there's no backends at all configured, we could use some tricks to + // automatically setup backends based if we have a TTY or not. + sequenceNo = 0 + b := SetBackend(NewLogBackend(os.Stderr, "", log.LstdFlags)) + b.SetLevel(DEBUG, "") + SetFormatter(DefaultFormatter) + timeNow = time.Now +} + +// InitForTesting is a convenient method when using logging in a test. Once +// called, the time will be frozen to January 1, 1970 UTC. +func InitForTesting(level Level) *MemoryBackend { + Reset() + + memoryBackend := NewMemoryBackend(10240) + + leveledBackend := AddModuleLevel(memoryBackend) + leveledBackend.SetLevel(level, "") + SetBackend(leveledBackend) + + timeNow = func() time.Time { + return time.Unix(0, 0).UTC() + } + return memoryBackend +} + +// IsEnabledFor returns true if the logger is enabled for the given level. +func (l *Logger) IsEnabledFor(level Level) bool { + return defaultBackend.IsEnabledFor(level, l.Module) +} + +func (l *Logger) log(lvl Level, format string, args ...interface{}) { + // Create the logging record and pass it in to the backend + record := &Record{ + Id: atomic.AddUint64(&sequenceNo, 1), + Time: timeNow(), + Module: l.Module, + Level: lvl, + fmt: format, + args: args, + } + + // TODO use channels to fan out the records to all backends? + // TODO in case of errors, do something (tricky) + + // calldepth=2 brings the stack up to the caller of the level + // methods, Info(), Fatal(), etc. + defaultBackend.Log(lvl, 2, record) +} + +// Fatal is equivalent to l.Critical(fmt.Sprint()) followed by a call to os.Exit(1). +func (l *Logger) Fatal(args ...interface{}) { + s := fmt.Sprint(args...) + l.log(CRITICAL, "%s", s) + os.Exit(1) +} + +// Fatalf is equivalent to l.Critical followed by a call to os.Exit(1). +func (l *Logger) Fatalf(format string, args ...interface{}) { + l.log(CRITICAL, format, args...) + os.Exit(1) +} + +// Panic is equivalent to l.Critical(fmt.Sprint()) followed by a call to panic(). +func (l *Logger) Panic(args ...interface{}) { + s := fmt.Sprint(args...) + l.log(CRITICAL, "%s", s) + panic(s) +} + +// Panicf is equivalent to l.Critical followed by a call to panic(). +func (l *Logger) Panicf(format string, args ...interface{}) { + s := fmt.Sprintf(format, args...) + l.log(CRITICAL, "%s", s) + panic(s) +} + +// Critical logs a message using CRITICAL as log level. +func (l *Logger) Critical(format string, args ...interface{}) { + l.log(CRITICAL, format, args...) +} + +// Error logs a message using ERROR as log level. +func (l *Logger) Error(format string, args ...interface{}) { + l.log(ERROR, format, args...) +} + +// Warning logs a message using WARNING as log level. +func (l *Logger) Warning(format string, args ...interface{}) { + l.log(WARNING, format, args...) +} + +// Notice logs a message using NOTICE as log level. +func (l *Logger) Notice(format string, args ...interface{}) { + l.log(NOTICE, format, args...) +} + +// Info logs a message using INFO as log level. +func (l *Logger) Info(format string, args ...interface{}) { + l.log(INFO, format, args...) +} + +// Debug logs a message using DEBUG as log level. +func (l *Logger) Debug(format string, args ...interface{}) { + l.log(DEBUG, format, args...) +} + +func init() { + Reset() +} diff --git a/Godeps/_workspace/src/github.com/op/go-logging/logger_test.go b/Godeps/_workspace/src/github.com/op/go-logging/logger_test.go new file mode 100644 index 000000000..ed295f660 --- /dev/null +++ b/Godeps/_workspace/src/github.com/op/go-logging/logger_test.go @@ -0,0 +1,36 @@ +// Copyright 2013, Örjan Persson. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package logging + +import "testing" + +type Password string + +func (p Password) Redacted() interface{} { + return Redact(string(p)) +} + +func TestSequenceNoOverflow(t *testing.T) { + // Forcefully set the next sequence number to the maximum + backend := InitForTesting(DEBUG) + sequenceNo = ^uint64(0) + + log := MustGetLogger("test") + log.Debug("test") + + if MemoryRecordN(backend, 0).Id != 0 { + t.Errorf("Unexpected sequence no: %v", MemoryRecordN(backend, 0).Id) + } +} + +func TestRedact(t *testing.T) { + backend := InitForTesting(DEBUG) + password := Password("123456") + log := MustGetLogger("test") + log.Debug("foo %s", password) + if "foo ******" != MemoryRecordN(backend, 0).Formatted(0) { + t.Errorf("redacted line: %v", MemoryRecordN(backend, 0)) + } +} diff --git a/Godeps/_workspace/src/github.com/op/go-logging/memory.go b/Godeps/_workspace/src/github.com/op/go-logging/memory.go new file mode 100644 index 000000000..a4f5ae436 --- /dev/null +++ b/Godeps/_workspace/src/github.com/op/go-logging/memory.go @@ -0,0 +1,217 @@ +// Copyright 2013, Örjan Persson. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package logging + +import ( + "sync" + "sync/atomic" + "unsafe" +) + +// TODO pick one of the memory backends and stick with it or share interface. + +// Node is a record node pointing to an optional next node. +type node struct { + next *node + Record *Record +} + +// Next returns the next record node. If there's no node available, it will +// return nil. +func (n *node) Next() *node { + return n.next +} + +// MemoryBackend is a simple memory based logging backend that will not produce +// any output but merly keep records, up to the given size, in memory. +type MemoryBackend struct { + size int32 + maxSize int32 + head, tail unsafe.Pointer +} + +// NewMemoryBackend creates a simple in-memory logging backend. +func NewMemoryBackend(size int) *MemoryBackend { + return &MemoryBackend{maxSize: int32(size)} +} + +// Log implements the Log method required by Backend. +func (b *MemoryBackend) Log(level Level, calldepth int, rec *Record) error { + var size int32 + + n := &node{Record: rec} + np := unsafe.Pointer(n) + + // Add the record to the tail. If there's no records available, tail and + // head will both be nil. When we successfully set the tail and the previous + // value was nil, it's safe to set the head to the current value too. + for { + tailp := b.tail + swapped := atomic.CompareAndSwapPointer( + &b.tail, + tailp, + np, + ) + if swapped == true { + if tailp == nil { + b.head = np + } else { + (*node)(tailp).next = n + } + size = atomic.AddInt32(&b.size, 1) + break + } + } + + // Since one record was added, we might have overflowed the list. Remove + // a record if that is the case. The size will fluctate a bit, but + // eventual consistent. + if b.maxSize > 0 && size > b.maxSize { + for { + headp := b.head + head := (*node)(b.head) + if head.next == nil { + break + } + swapped := atomic.CompareAndSwapPointer( + &b.head, + headp, + unsafe.Pointer(head.next), + ) + if swapped == true { + atomic.AddInt32(&b.size, -1) + break + } + } + } + return nil +} + +// Head returns the oldest record node kept in memory. It can be used to +// iterate over records, one by one, up to the last record. +// +// Note: new records can get added while iterating. Hence the number of records +// iterated over might be larger than the maximum size. +func (b *MemoryBackend) Head() *node { + return (*node)(b.head) +} + +type event int + +const ( + eventFlush event = iota + eventStop +) + +// ChannelMemoryBackend is very similar to the MemoryBackend, except that it +// internally utilizes a channel. +type ChannelMemoryBackend struct { + maxSize int + size int + incoming chan *Record + events chan event + mu sync.Mutex + running bool + flushWg sync.WaitGroup + stopWg sync.WaitGroup + head, tail *node +} + +// NewChannelMemoryBackend creates a simple in-memory logging backend which +// utilizes a go channel for communication. +// +// Start will automatically be called by this function. +func NewChannelMemoryBackend(size int) *ChannelMemoryBackend { + backend := &ChannelMemoryBackend{ + maxSize: size, + incoming: make(chan *Record, 1024), + events: make(chan event), + } + backend.Start() + return backend +} + +// Start launches the internal goroutine which starts processing data from the +// input channel. +func (b *ChannelMemoryBackend) Start() { + b.mu.Lock() + defer b.mu.Unlock() + + // Launch the goroutine unless it's already running. + if b.running != true { + b.running = true + b.stopWg.Add(1) + go b.process() + } +} + +func (b *ChannelMemoryBackend) process() { + for { + select { + case rec := <-b.incoming: + b.insertRecord(rec) + case e := <-b.events: + switch e { + case eventStop: + break + case eventFlush: + for len(b.incoming) > 0 { + b.insertRecord(<-b.incoming) + } + b.flushWg.Done() + } + } + } + b.stopWg.Done() +} + +func (b *ChannelMemoryBackend) insertRecord(rec *Record) { + prev := b.tail + b.tail = &node{Record: rec} + if prev == nil { + b.head = b.tail + } else { + prev.next = b.tail + } + + if b.maxSize > 0 && b.size >= b.maxSize { + b.head = b.head.next + } else { + b.size += 1 + } +} + +// Flush waits until all records in the buffered channel have been processed. +func (b *ChannelMemoryBackend) Flush() { + b.flushWg.Add(1) + b.events <- eventFlush + b.flushWg.Wait() +} + +// Stop signals the internal goroutine to exit and waits until it have. +func (b *ChannelMemoryBackend) Stop() { + b.mu.Lock() + if b.running == true { + b.running = false + b.events <- eventStop + } + b.mu.Unlock() + b.stopWg.Wait() +} + +// Log implements the Log method required by Backend. +func (b *ChannelMemoryBackend) Log(level Level, calldepth int, rec *Record) error { + b.incoming <- rec + return nil +} + +// Head returns the oldest record node kept in memory. It can be used to +// iterate over records, one by one, up to the last record. +// +// Note: new records can get added while iterating. Hence the number of records +// iterated over might be larger than the maximum size. +func (b *ChannelMemoryBackend) Head() *node { + return b.head +} diff --git a/Godeps/_workspace/src/github.com/op/go-logging/memory_test.go b/Godeps/_workspace/src/github.com/op/go-logging/memory_test.go new file mode 100644 index 000000000..fe5a82e44 --- /dev/null +++ b/Godeps/_workspace/src/github.com/op/go-logging/memory_test.go @@ -0,0 +1,117 @@ +// Copyright 2013, Örjan Persson. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package logging + +import ( + "strconv" + "testing" +) + +// TODO share more code between these tests +func MemoryRecordN(b *MemoryBackend, n int) *Record { + node := b.Head() + for i := 0; i < n; i++ { + if node == nil { + break + } + node = node.Next() + } + if node == nil { + return nil + } + return node.Record +} + +func ChannelMemoryRecordN(b *ChannelMemoryBackend, n int) *Record { + b.Flush() + node := b.Head() + for i := 0; i < n; i++ { + if node == nil { + break + } + node = node.Next() + } + if node == nil { + return nil + } + return node.Record +} + +func TestMemoryBackend(t *testing.T) { + backend := NewMemoryBackend(8) + SetBackend(backend) + + log := MustGetLogger("test") + + if nil != MemoryRecordN(backend, 0) || 0 != backend.size { + t.Errorf("memory level: %d", backend.size) + } + + // Run 13 times, the resulting vector should be [5..12] + for i := 0; i < 13; i++ { + log.Info("%d", i) + } + + if 8 != backend.size { + t.Errorf("record length: %d", backend.size) + } + record := MemoryRecordN(backend, 0) + if "5" != record.Formatted(0) { + t.Errorf("unexpected start: %s", record.Formatted(0)) + } + for i := 0; i < 8; i++ { + record = MemoryRecordN(backend, i) + if strconv.Itoa(i+5) != record.Formatted(0) { + t.Errorf("unexpected record: %v", record.Formatted(0)) + } + } + record = MemoryRecordN(backend, 7) + if "12" != record.Formatted(0) { + t.Errorf("unexpected end: %s", record.Formatted(0)) + } + record = MemoryRecordN(backend, 8) + if nil != record { + t.Errorf("unexpected eof: %s", record.Formatted(0)) + } +} + +func TestChannelMemoryBackend(t *testing.T) { + backend := NewChannelMemoryBackend(8) + SetBackend(backend) + + log := MustGetLogger("test") + + if nil != ChannelMemoryRecordN(backend, 0) || 0 != backend.size { + t.Errorf("memory level: %d", backend.size) + } + + // Run 13 times, the resulting vector should be [5..12] + for i := 0; i < 13; i++ { + log.Info("%d", i) + } + backend.Flush() + + if 8 != backend.size { + t.Errorf("record length: %d", backend.size) + } + record := ChannelMemoryRecordN(backend, 0) + if "5" != record.Formatted(0) { + t.Errorf("unexpected start: %s", record.Formatted(0)) + } + for i := 0; i < 8; i++ { + record = ChannelMemoryRecordN(backend, i) + if strconv.Itoa(i+5) != record.Formatted(0) { + t.Errorf("unexpected record: %v", record.Formatted(0)) + } + } + record = ChannelMemoryRecordN(backend, 7) + if "12" != record.Formatted(0) { + t.Errorf("unexpected end: %s", record.Formatted(0)) + } + record = ChannelMemoryRecordN(backend, 8) + if nil != record { + t.Errorf("unexpected eof: %s", record.Formatted(0)) + } +} diff --git a/Godeps/_workspace/src/github.com/op/go-logging/multi.go b/Godeps/_workspace/src/github.com/op/go-logging/multi.go new file mode 100644 index 000000000..75a3e237a --- /dev/null +++ b/Godeps/_workspace/src/github.com/op/go-logging/multi.go @@ -0,0 +1,63 @@ +// Copyright 2013, Örjan Persson. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package logging + +// multiLogger is a log multiplexer which can be used to utilize multiple log +// backends at once. +type multiLogger struct { + backends []LeveledBackend +} + +// MultiLogger creates a logger which contain multiple loggers. +func MultiLogger(backends ...Backend) LeveledBackend { + var leveledBackends []LeveledBackend + for _, backend := range backends { + leveledBackends = append(leveledBackends, AddModuleLevel(backend)) + } + return &multiLogger{leveledBackends} +} + +// Log passes the log record to all backends. +func (b *multiLogger) Log(level Level, calldepth int, rec *Record) (err error) { + for _, backend := range b.backends { + if backend.IsEnabledFor(level, rec.Module) { + // Shallow copy of the record for the formatted cache on Record and get the + // record formatter from the backend. + r2 := *rec + if e := backend.Log(level, calldepth+1, &r2); e != nil { + err = e + } + } + } + return +} + +// GetLevel returns the highest level enabled by all backends. +func (b *multiLogger) GetLevel(module string) Level { + var level Level + for _, backend := range b.backends { + if backendLevel := backend.GetLevel(module); backendLevel > level { + level = backendLevel + } + } + return level +} + +// SetLevel propagates the same level to all backends. +func (b *multiLogger) SetLevel(level Level, module string) { + for _, backend := range b.backends { + backend.SetLevel(level, module) + } +} + +// IsEnabledFor returns true if any of the backends are enabled for it. +func (b *multiLogger) IsEnabledFor(level Level, module string) bool { + for _, backend := range b.backends { + if backend.IsEnabledFor(level, module) { + return true + } + } + return false +} diff --git a/Godeps/_workspace/src/github.com/op/go-logging/multi_test.go b/Godeps/_workspace/src/github.com/op/go-logging/multi_test.go new file mode 100644 index 000000000..b6ecf5b5b --- /dev/null +++ b/Godeps/_workspace/src/github.com/op/go-logging/multi_test.go @@ -0,0 +1,51 @@ +// Copyright 2013, Örjan Persson. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package logging + +import "testing" + +func TestMultiLogger(t *testing.T) { + log1 := NewMemoryBackend(8) + log2 := NewMemoryBackend(8) + SetBackend(MultiLogger(log1, log2)) + + log := MustGetLogger("test") + log.Debug("log") + + if "log" != MemoryRecordN(log1, 0).Formatted(0) { + t.Errorf("log1: %v", MemoryRecordN(log1, 0).Formatted(0)) + } + if "log" != MemoryRecordN(log2, 0).Formatted(0) { + t.Errorf("log2: %v", MemoryRecordN(log2, 0).Formatted(0)) + } +} + +func TestMultiLoggerLevel(t *testing.T) { + log1 := NewMemoryBackend(8) + log2 := NewMemoryBackend(8) + + leveled1 := AddModuleLevel(log1) + leveled2 := AddModuleLevel(log2) + + multi := MultiLogger(leveled1, leveled2) + multi.SetLevel(ERROR, "test") + SetBackend(multi) + + log := MustGetLogger("test") + log.Notice("log") + + if nil != MemoryRecordN(log1, 0) || nil != MemoryRecordN(log2, 0) { + t.Errorf("unexpected log record") + } + + leveled1.SetLevel(DEBUG, "test") + log.Notice("log") + if "log" != MemoryRecordN(log1, 0).Formatted(0) { + t.Errorf("log1 not receieved") + } + if nil != MemoryRecordN(log2, 0) { + t.Errorf("log2 receieved") + } +} diff --git a/Godeps/_workspace/src/github.com/op/go-logging/syslog.go b/Godeps/_workspace/src/github.com/op/go-logging/syslog.go new file mode 100644 index 000000000..9a5c8f5d3 --- /dev/null +++ b/Godeps/_workspace/src/github.com/op/go-logging/syslog.go @@ -0,0 +1,52 @@ +// Copyright 2013, Örjan Persson. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//+build !windows,!plan9 + +package logging + +import "log/syslog" + +// SyslogBackend is a simple logger to syslog backend. It automatically maps +// the internal log levels to appropriate syslog log levels. +type SyslogBackend struct { + Writer *syslog.Writer +} + +// NewSyslogBackend connects to the syslog daemon using UNIX sockets with the +// given prefix. If prefix is not given, the prefix will be derived from the +// launched command. +func NewSyslogBackend(prefix string) (b *SyslogBackend, err error) { + var w *syslog.Writer + w, err = syslog.New(syslog.LOG_CRIT, prefix) + return &SyslogBackend{w}, err +} + +// NewSyslogBackendPriority is the same as NewSyslogBackend, but with custom +// syslog priority, like syslog.LOG_LOCAL3|syslog.LOG_DEBUG etc. +func NewSyslogBackendPriority(prefix string, priority syslog.Priority) (b *SyslogBackend, err error) { + var w *syslog.Writer + w, err = syslog.New(priority, prefix) + return &SyslogBackend{w}, err +} + +func (b *SyslogBackend) Log(level Level, calldepth int, rec *Record) error { + line := rec.Formatted(calldepth + 1) + switch level { + case CRITICAL: + return b.Writer.Crit(line) + case ERROR: + return b.Writer.Err(line) + case WARNING: + return b.Writer.Warning(line) + case NOTICE: + return b.Writer.Notice(line) + case INFO: + return b.Writer.Info(line) + case DEBUG: + return b.Writer.Debug(line) + default: + } + panic("unhandled log level") +} diff --git a/daemon/daemon.go b/daemon/daemon.go index 6ec8f9d1e..8180131ba 100644 --- a/daemon/daemon.go +++ b/daemon/daemon.go @@ -9,12 +9,12 @@ import ( "path" "sync" + logging "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/op/go-logging" core "github.com/jbenet/go-ipfs/core" "github.com/jbenet/go-ipfs/core/commands" u "github.com/jbenet/go-ipfs/util" - logging "github.com/op/go-logging" - lock "github.com/camlistore/lock" + lock "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/camlistore/lock" ma "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multiaddr" )