/[gentoo]/xml/htdocs/proj/en/glep/glep-0044.html
Gentoo

Diff of /xml/htdocs/proj/en/glep/glep-0044.html

Parent Directory Parent Directory | Revision Log Revision Log | View Patch Patch

Revision 1.4 Revision 1.10
1<?xml version="1.0" encoding="utf-8" ?> 1<?xml version="1.0" encoding="utf-8" ?>
2<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"> 2<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
3<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en"> 3<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
4<!-- 4
5This HTML is auto-generated. DO NOT EDIT THIS FILE! If you are writing a new
6PEP, see http://www.python.org/peps/pep-0001.html for instructions and links
7to templates. DO NOT USE THIS HTML FILE AS YOUR TEMPLATE!
8-->
9<head> 5<head>
10 <meta http-equiv="Content-Type" content="text/html; charset=utf-8" /> 6 <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
11 <meta name="generator" content="Docutils 0.3.9: http://docutils.sourceforge.net/" /> 7 <meta name="generator" content="Docutils 0.4: http://docutils.sourceforge.net/" />
12 <title>GLEP 44 -- Manifest2 format</title> 8 <title>GLEP 44 -- Manifest2 format</title>
13 <link rel="stylesheet" href="tools/glep.css" type="text/css" /> 9 <link rel="stylesheet" href="tools/glep.css" type="text/css" />
14</head> 10</head>
15<body bgcolor="white"> 11<body bgcolor="white">
16<table class="navigation" cellpadding="0" cellspacing="0" 12<table class="navigation" cellpadding="0" cellspacing="0"
20<img src="http://www.gentoo.org/images/gentoo-new.gif" alt="[Gentoo]" 16<img src="http://www.gentoo.org/images/gentoo-new.gif" alt="[Gentoo]"
21 border="0" width="150" height="35" /></a></td> 17 border="0" width="150" height="35" /></a></td>
22<td class="textlinks" align="left"> 18<td class="textlinks" align="left">
23[<b><a href="http://www.gentoo.org/">Gentoo Linux Home</a></b>] 19[<b><a href="http://www.gentoo.org/">Gentoo Linux Home</a></b>]
24[<b><a href="http://www.gentoo.org/proj/en/glep">GLEP Index</a></b>] 20[<b><a href="http://www.gentoo.org/proj/en/glep">GLEP Index</a></b>]
25[<b><a href="./glep-0044.txt">GLEP Source</a></b>] 21[<b><a href="http://www.gentoo.org/proj/en/glep/glep-0044.txt">GLEP Source</a></b>]
26</td></tr></table> 22</td></tr></table>
27<table class="rfc2822 docutils field-list" frame="void" rules="none"> 23<table class="rfc2822 docutils field-list" frame="void" rules="none">
28<col class="field-name" /> 24<col class="field-name" />
29<col class="field-body" /> 25<col class="field-body" />
30<tbody valign="top"> 26<tbody valign="top">
31<tr class="field"><th class="field-name">GLEP:</th><td class="field-body">44</td> 27<tr class="field"><th class="field-name">GLEP:</th><td class="field-body">44</td>
32</tr> 28</tr>
33<tr class="field"><th class="field-name">Title:</th><td class="field-body">Manifest2 format</td> 29<tr class="field"><th class="field-name">Title:</th><td class="field-body">Manifest2 format</td>
34</tr> 30</tr>
35<tr class="field"><th class="field-name">Version:</th><td class="field-body">1.3</td> 31<tr class="field"><th class="field-name">Version:</th><td class="field-body">1.7</td>
36</tr> 32</tr>
37<tr class="field"><th class="field-name">Last-Modified:</th><td class="field-body"><a class="reference" href="http://www.gentoo.org/cgi-bin/viewcvs/xml/htdocs/proj/en/glep/glep-0044.txt?cvsroot=gentoo">2006/01/23 10:24:24</a></td> 33<tr class="field"><th class="field-name">Last-Modified:</th><td class="field-body"><a class="reference" href="http://www.gentoo.org/cgi-bin/viewcvs.cgi/xml/htdocs/proj/en/glep/glep-0044.txt?cvsroot=gentoo">2006/10/14 02:55:39</a></td>
38</tr> 34</tr>
39<tr class="field"><th class="field-name">Author:</th><td class="field-body">Marius Mauch &lt;genone&#32;&#97;t&#32;gentoo.org&gt;,</td> 35<tr class="field"><th class="field-name">Author:</th><td class="field-body">Marius Mauch &lt;genone&#32;&#97;t&#32;gentoo.org&gt;,</td>
40</tr> 36</tr>
41<tr class="field"><th class="field-name">Status:</th><td class="field-body">Draft</td> 37<tr class="field"><th class="field-name">Status:</th><td class="field-body">Accepted</td>
42</tr> 38</tr>
43<tr class="field"><th class="field-name">Type:</th><td class="field-body">Standards Track</td> 39<tr class="field"><th class="field-name">Type:</th><td class="field-body">Standards Track</td>
44</tr> 40</tr>
45<tr class="field"><th class="field-name">Content-Type:</th><td class="field-body"><a class="reference" href="http://www.python.org/peps/glep-0012.html">text/x-rst</a></td> 41<tr class="field"><th class="field-name">Content-Type:</th><td class="field-body"><a class="reference" href="glep-0002.html">text/x-rst</a></td>
46</tr> 42</tr>
47<tr class="field"><th class="field-name">Created:</th><td class="field-body">04-Dec-2005</td> 43<tr class="field"><th class="field-name">Created:</th><td class="field-body">04-Dec-2005</td>
48</tr> 44</tr>
49<tr class="field"><th class="field-name">Post-History:</th><td class="field-body">06-Dec-2005, 23-Jan-2006</td> 45<tr class="field"><th class="field-name">Post-History:</th><td class="field-body">06-Dec-2005, 23-Jan-2006, 3-Sep-2006</td>
50</tr> 46</tr>
51</tbody> 47</tbody>
52</table> 48</table>
53<hr /> 49<hr />
54<div class="contents topic" id="contents"> 50<div class="contents topic">
55<p class="topic-title first"><a name="contents">Contents</a></p> 51<p class="topic-title first"><a id="contents" name="contents">Contents</a></p>
56<ul class="simple"> 52<ul class="simple">
57<li><a class="reference" href="#abstract" id="id9" name="id9">Abstract</a></li> 53<li><a class="reference" href="#abstract" id="id9" name="id9">Abstract</a></li>
58<li><a class="reference" href="#motivation" id="id10" name="id10">Motivation</a></li> 54<li><a class="reference" href="#motivation" id="id10" name="id10">Motivation</a></li>
59<li><a class="reference" href="#specification" id="id11" name="id11">Specification</a><ul> 55<li><a class="reference" href="#specification" id="id11" name="id11">Specification</a><ul>
60<li><a class="reference" href="#compability-entries" id="id12" name="id12">Compability Entries</a></li> 56<li><a class="reference" href="#compability-entries" id="id12" name="id12">Compability Entries</a></li>
79<li><a class="reference" href="#credits" id="id25" name="id25">Credits</a></li> 75<li><a class="reference" href="#credits" id="id25" name="id25">Credits</a></li>
80<li><a class="reference" href="#references" id="id26" name="id26">References</a></li> 76<li><a class="reference" href="#references" id="id26" name="id26">References</a></li>
81<li><a class="reference" href="#copyright" id="id27" name="id27">Copyright</a></li> 77<li><a class="reference" href="#copyright" id="id27" name="id27">Copyright</a></li>
82</ul> 78</ul>
83</div> 79</div>
84<div class="section" id="abstract"> 80<div class="section">
85<h1><a class="toc-backref" href="#id9" name="abstract">Abstract</a></h1> 81<h1><a class="toc-backref" href="#id9" id="abstract" name="abstract">Abstract</a></h1>
86<p>This GLEP proposes a new format for the Portage Manifest and digest file system 82<p>This GLEP proposes a new format for the Portage Manifest and digest file system
87by unifying both filetypes into one to improve functional and non-functional 83by unifying both filetypes into one to improve functional and non-functional
88aspects of the Portage Tree.</p> 84aspects of the Portage Tree.</p>
89</div> 85</div>
90<div class="section" id="motivation"> 86<div class="section">
91<h1><a class="toc-backref" href="#id10" name="motivation">Motivation</a></h1> 87<h1><a class="toc-backref" href="#id10" id="motivation" name="motivation">Motivation</a></h1>
92<p>Please see <a class="footnote-reference" href="#reorg-thread" id="id1" name="id1">[1]</a> for a general overview. 88<p>Please see <a class="footnote-reference" href="#reorg-thread" id="id1" name="id1">[1]</a> for a general overview.
93The main long term goals of this proposal are to:</p> 89The main long term goals of this proposal are to:</p>
94<ul class="simple"> 90<ul class="simple">
95<li>Remove the tiny digest files from the tree. They are a major annoyance as on a 91<li>Remove the tiny digest files from the tree. They are a major annoyance as on a
96typical configuration they waste a lot of disk space and the simple transmission 92typical configuration they waste a lot of disk space and the simple transmission
100<li>Remove potential for checksum collisions if a file is recorded in more than one 96<li>Remove potential for checksum collisions if a file is recorded in more than one
101digest file</li> 97digest file</li>
102<li>Difference between filetypes for a more flexible verification system</li> 98<li>Difference between filetypes for a more flexible verification system</li>
103</ul> 99</ul>
104</div> 100</div>
105<div class="section" id="specification"> 101<div class="section">
106<h1><a class="toc-backref" href="#id11" name="specification">Specification</a></h1> 102<h1><a class="toc-backref" href="#id11" id="specification" name="specification">Specification</a></h1>
107<p>The new Manifest format would change the existing format in the following ways:</p> 103<p>The new Manifest format would change the existing format in the following ways:</p>
108<ul> 104<ul>
109<li><p class="first">Addition of a filetype specifier, currently planned are</p> 105<li><p class="first">Addition of a filetype specifier, currently planned are</p>
110<ul class="simple"> 106<ul class="simple">
111<li><tt class="docutils literal"><span class="pre">AUX</span></tt> for files directly used by ebuilds (e.g. patches or initscripts), 107<li><tt class="docutils literal"><span class="pre">AUX</span></tt> for files directly used by ebuilds (e.g. patches or initscripts),
114<li><tt class="docutils literal"><span class="pre">MISC</span></tt> for files not directly used by ebuilds like <tt class="docutils literal"><span class="pre">ChangeLog</span></tt> or 110<li><tt class="docutils literal"><span class="pre">MISC</span></tt> for files not directly used by ebuilds like <tt class="docutils literal"><span class="pre">ChangeLog</span></tt> or
115<tt class="docutils literal"><span class="pre">metadata.xml</span></tt> files</li> 111<tt class="docutils literal"><span class="pre">metadata.xml</span></tt> files</li>
116<li><tt class="docutils literal"><span class="pre">DIST</span></tt> for release tarballs recorded in the <tt class="docutils literal"><span class="pre">SRC_URI</span></tt> variable of an ebuild, 112<li><tt class="docutils literal"><span class="pre">DIST</span></tt> for release tarballs recorded in the <tt class="docutils literal"><span class="pre">SRC_URI</span></tt> variable of an ebuild,
117these were previously recorded in the digest files</li> 113these were previously recorded in the digest files</li>
118</ul> 114</ul>
119<p>Future portage improvements might extend this list (for example with types 115<p>Future portage improvements might extend this list (for example with types
120relevant for eclasses or profiles)</p> 116relevant for eclasses or profiles)</p>
121</li> 117</li>
122<li><p class="first">Only have one line per file listing all information instead of one line per 118<li><p class="first">Only have one line per file listing all information instead of one line per
123file and checksum type</p> 119file and checksum type</p>
124</li> 120</li>
125<li><p class="first">Remove the separated digest-* files in the <tt class="docutils literal"><span class="pre">files/</span></tt> subdirectory</p> 121<li><p class="first">Remove the separated digest-* files in the <tt class="docutils literal"><span class="pre">files/</span></tt> subdirectory</p>
126</li> 122</li>
127</ul> 123</ul>
128<p>Each line in the new format has the following format:</p> 124<p>Each line in the new format has the following format:</p>
129<pre class="literal-block"> 125<pre class="literal-block">
130&lt;filetype&gt; &lt;filename&gt; &lt;filesize&gt; &lt;chksumtype1&gt; &lt;chksum1&gt; ... &lt;chksumtypen&gt; &lt;chksumn&gt; 126&lt;filetype&gt; &lt;filename&gt; &lt;filesize&gt; &lt;chksumtype1&gt; &lt;chksum1&gt; ... &lt;chksumtypen&gt; &lt;chksumn&gt;
131</pre> 127</pre>
132<p>However theses entries will be stored in the existing Manifest files.</p> 128<p>However theses entries will be stored in the existing Manifest files.</p>
133<p>An <a class="reference" href="glep-0044-extras/manifest2-example">actual example</a> <a class="footnote-reference" href="#id7" id="id8" name="id8">[6]</a> for a (pure) Manifest2 file..</p> 129<p>An <a class="reference" href="glep-0044-extras/manifest2-example.txt">actual example</a> <a class="footnote-reference" href="#id7" id="id8" name="id8">[6]</a> for a (pure) Manifest2 file..</p>
134<div class="section" id="compability-entries"> 130<div class="section">
135<h2><a class="toc-backref" href="#id12" name="compability-entries">Compability Entries</a></h2> 131<h2><a class="toc-backref" href="#id12" id="compability-entries" name="compability-entries">Compability Entries</a></h2>
136<p>To maintain compability with existing portage versions a transition period after 132<p>To maintain compability with existing portage versions a transition period after
137is the introduction of the Manifest2 format is required during which portage 133is the introduction of the Manifest2 format is required during which portage
138will not only have to be capable of using existing Manifest and digest files but 134will not only have to be capable of using existing Manifest and digest files but
139also generate them in addition to the new entries. 135also generate them in addition to the new entries.
140Fortunately this can be accomplished by simply mixing old and new style entries 136Fortunately this can be accomplished by simply mixing old and new style entries
141in one file for the Manifest files, existing portage versions will simply ignore 137in one file for the Manifest files, existing portage versions will simply ignore
142the new style entries. For the digest files there are no new entries to care 138the new style entries. For the digest files there are no new entries to care
143about.</p> 139about.</p>
144</div> 140</div>
145<div class="section" id="scope"> 141<div class="section">
146<h2><a class="toc-backref" href="#id13" name="scope">Scope</a></h2> 142<h2><a class="toc-backref" href="#id13" id="scope" name="scope">Scope</a></h2>
147<p>It is important to note that this proposal only deals with a change of the 143<p>It is important to note that this proposal only deals with a change of the
148format of the digest and Manifest system.</p> 144format of the digest and Manifest system.</p>
149<p>It does not expand the scope of it to cover eclasses, profiles or anything 145<p>It does not expand the scope of it to cover eclasses, profiles or anything
150else not already covered by the Manifest system, it also doesn't affect 146else not already covered by the Manifest system, it also doesn't affect
151the Manifest signing efforts in any way (though the implementations of both 147the Manifest signing efforts in any way (though the implementations of both
152might be coupled).</p> 148might be coupled).</p>
153<p>Also while multiple hash functions will become standard with the proposed 149<p>Also while multiple hash functions will become standard with the proposed
154implementation they are not a specific feature of this format <a class="footnote-reference" href="#multi-hash-thread" id="id3" name="id3">[2]</a>.</p> 150implementation they are not a specific feature of this format <a class="footnote-reference" href="#multi-hash-thread" id="id3" name="id3">[2]</a>.</p>
155</div> 151</div>
156<div class="section" id="number-of-hashes"> 152<div class="section">
157<h2><a class="toc-backref" href="#id14" name="number-of-hashes">Number of hashes</a></h2> 153<h2><a class="toc-backref" href="#id14" id="number-of-hashes" name="number-of-hashes">Number of hashes</a></h2>
158<p>While using multiple hashes for each file is a major feature of this proposal 154<p>While using multiple hashes for each file is a major feature of this proposal
159we have to make sure that the number of hashes listed is limited to avoid 155we have to make sure that the number of hashes listed is limited to avoid
160an explosion of the Manifest size that would revert the main benefit of this proposal 156an explosion of the Manifest size that would revert the main benefit of this proposal
161(reduzing tree size). Therefore the number of hashes that will be generated 157(reduzing tree size). Therefore the number of hashes that will be generated
162will be limited to three different hash functions. For compability though we 158will be limited to three different hash functions. For compability though we
163have to rely on at least one hash function to always be present, this proposal 159have to rely on at least one hash function to always be present, this proposal
164suggest to use SHA1 for this purpose (as it is supposed to be more secure than MD5 160suggest to use SHA1 for this purpose (as it is supposed to be more secure than MD5
165and currently only SHA1 and MD5 are directly available in python, also MD5 doesn't 161and currently only SHA1 and MD5 are directly available in python, also MD5 doesn't
166have any benefit in terms of compability).</p> 162have any benefit in terms of compability).</p>
167</div> 163</div>
168</div> 164</div>
169<div class="section" id="rationale"> 165<div class="section">
170<h1><a class="toc-backref" href="#id15" name="rationale">Rationale</a></h1> 166<h1><a class="toc-backref" href="#id15" id="rationale" name="rationale">Rationale</a></h1>
171<p>The main goals of the proposal have been listed in the <a class="reference" href="#motivation">Motivation</a>, here now 167<p>The main goals of the proposal have been listed in the <a class="reference" href="#motivation">Motivation</a>, here now
172the explanation why they are improvements and how the proposed format will 168the explanation why they are improvements and how the proposed format will
173accomplish them.</p> 169accomplish them.</p>
174<div class="section" id="removal-of-digest-files"> 170<div class="section">
175<h2><a class="toc-backref" href="#id16" name="removal-of-digest-files">Removal of digest files</a></h2> 171<h2><a class="toc-backref" href="#id16" id="removal-of-digest-files" name="removal-of-digest-files">Removal of digest files</a></h2>
176<p>Normal users that don't use a &quot;tuned&quot; filesystem for the portage tree are 172<p>Normal users that don't use a &quot;tuned&quot; filesystem for the portage tree are
177wasting several dozen to a few hundred megabytes of disk space with the current 173wasting several dozen to a few hundred megabytes of disk space with the current
178system, largely caused by the digest files. 174system, largely caused by the digest files.
179This is due to the filesystem overhead present in most filesystem that 175This is due to the filesystem overhead present in most filesystem that
180have a standard blocksize of four kilobytes while most digest files are under 176have a standard blocksize of four kilobytes while most digest files are under
181one kilobyte in size, so this results in approximately a waste of three kilobytes 177one kilobyte in size, so this results in approximately a waste of three kilobytes
182per digest file (likely even more). At the time of this writing the tree contains 178per digest file (likely even more). At the time of this writing the tree contains
183roughly 22.000 digest files, so the overall waste caused by digest files is 179roughly 22.000 digest files, so the overall waste caused by digest files is
184estimated at about 70-100 megabytes. 180estimated at about 70-100 megabytes.
185Furthermore it is assumed that this will also reduce the disk space wasted by 181Furthermore it is assumed that this will also reduce the disk space wasted by
186the Manifest files as they now contain more content, but this hasn't been 182the Manifest files as they now contain more content, but this hasn't been
187verified yet.</p> 183verified yet.</p>
188<p>By unifying the digest files with the Manifest these tiny files are eliminated 184<p>By unifying the digest files with the Manifest these tiny files are eliminated
189(in the long run), reducing the apparent tree size by about 20%, benefitting 185(in the long run), reducing the apparent tree size by about 20%, benefitting
190both users and the Gentoo infrastructure.</p> 186both users and the Gentoo infrastructure.</p>
191</div> 187</div>
192<div class="section" id="reducing-redundancy"> 188<div class="section">
193<h2><a class="toc-backref" href="#id17" name="reducing-redundancy">Reducing redundancy</a></h2> 189<h2><a class="toc-backref" href="#id17" id="reducing-redundancy" name="reducing-redundancy">Reducing redundancy</a></h2>
194<p>When multiple hashes are used with the current system 190<p>When multiple hashes are used with the current system
195both the filename and filesize are repeated for every checksum type used as each 191both the filename and filesize are repeated for every checksum type used as each
196checksum is standalone. However this doesn't add any functionality and is 192checksum is standalone. However this doesn't add any functionality and is
197therefore useless, so the new format removes this redundancy. 193therefore useless, so the new format removes this redundancy.
198This is a theoretical improvement at this moment as only one hash function is in 194This is a theoretical improvement at this moment as only one hash function is in
199use, but expected to change soon (see <a class="footnote-reference" href="#multi-hash-thread" id="id4" name="id4">[2]</a>).</p> 195use, but expected to change soon (see <a class="footnote-reference" href="#multi-hash-thread" id="id4" name="id4">[2]</a>).</p>
200</div> 196</div>
201<div class="section" id="removal-of-checksum-collisions"> 197<div class="section">
202<h2><a class="toc-backref" href="#id18" name="removal-of-checksum-collisions">Removal of checksum collisions</a></h2> 198<h2><a class="toc-backref" href="#id18" id="removal-of-checksum-collisions" name="removal-of-checksum-collisions">Removal of checksum collisions</a></h2>
203<p>The current system theoretically allows for a <tt class="docutils literal"><span class="pre">DIST</span></tt> type file to be recorded 199<p>The current system theoretically allows for a <tt class="docutils literal"><span class="pre">DIST</span></tt> type file to be recorded
204in multiple digest files with different sizes and/or checksums. In such a case 200in multiple digest files with different sizes and/or checksums. In such a case
205one version of a package would report a checksum violation while another one 201one version of a package would report a checksum violation while another one
206would not. This could create confusion and uncertainity among users. 202would not. This could create confusion and uncertainity among users.
207So far this case hasn't been observed, but it can't be ruled out with the 203So far this case hasn't been observed, but it can't be ruled out with the
208existing system. 204existing system.
209As the new format lists each file exactly once this would be no longer possible.</p> 205As the new format lists each file exactly once this would be no longer possible.</p>
210</div> 206</div>
211<div class="section" id="flexible-verification-system"> 207<div class="section">
212<h2><a class="toc-backref" href="#id19" name="flexible-verification-system">Flexible verification system</a></h2> 208<h2><a class="toc-backref" href="#id19" id="flexible-verification-system" name="flexible-verification-system">Flexible verification system</a></h2>
213<p>Right now portage verifies the checksum of every file listed in the Manifest 209<p>Right now portage verifies the checksum of every file listed in the Manifest
214before using any file of the package and all <tt class="docutils literal"><span class="pre">DIST</span></tt> files of an ebuild 210before using any file of the package and all <tt class="docutils literal"><span class="pre">DIST</span></tt> files of an ebuild
215before using that ebuild. This is unnecessary in many cases:</p> 211before using that ebuild. This is unnecessary in many cases:</p>
216<ul class="simple"> 212<ul class="simple">
217<li>During the &quot;depend&quot; phase (when the ebuild metadata is generated) only 213<li>During the &quot;depend&quot; phase (when the ebuild metadata is generated) only
218files of type <tt class="docutils literal"><span class="pre">EBUILD</span></tt> are used, so verifying the other types isn't 214files of type <tt class="docutils literal"><span class="pre">EBUILD</span></tt> are used, so verifying the other types isn't
219necessary. Theoretically it is possible for an ebuild to include other 215necessary. Theoretically it is possible for an ebuild to include other
220files like those of type <tt class="docutils literal"><span class="pre">AUX</span></tt> at this phase, but that would be a 216files like those of type <tt class="docutils literal"><span class="pre">AUX</span></tt> at this phase, but that would be a
221major QA violation and should never occur, so it can be ignored here. 217major QA violation and should never occur, so it can be ignored here.
222It is also not a security concern as the ebuild is verified before parsing 218It is also not a security concern as the ebuild is verified before parsing
223it, so each manipulation would show up.</li> 219it, so each manipulation would show up.</li>
224<li>Generally files of type <tt class="docutils literal"><span class="pre">MISC</span></tt> don't need to be verified as they are 220<li>Generally files of type <tt class="docutils literal"><span class="pre">MISC</span></tt> don't need to be verified as they are
225only used in very specific situations, aren't executed (just parsed at most) 221only used in very specific situations, aren't executed (just parsed at most)
226and don't affect the package build process.</li> 222and don't affect the package build process.</li>
227<li>Files of type <tt class="docutils literal"><span class="pre">DIST</span></tt> only need to be verified directly after fetching and 223<li>Files of type <tt class="docutils literal"><span class="pre">DIST</span></tt> only need to be verified directly after fetching and
228before unpacking them (which often will be one step), not every time their 224before unpacking them (which often will be one step), not every time their
229associated ebuild is used.</li> 225associated ebuild is used.</li>
230</ul> 226</ul>
231</div> 227</div>
232</div> 228</div>
233<div class="section" id="backwards-compatibility"> 229<div class="section">
234<h1><a class="toc-backref" href="#id20" name="backwards-compatibility">Backwards Compatibility</a></h1> 230<h1><a class="toc-backref" href="#id20" id="backwards-compatibility" name="backwards-compatibility">Backwards Compatibility</a></h1>
235<p>Switching the Manifest system is a task that will need a long transition period 231<p>Switching the Manifest system is a task that will need a long transition period
236like most changes affecting both portage and the tree. In this case the 232like most changes affecting both portage and the tree. In this case the
237implementation will be rolled out in several phases:</p> 233implementation will be rolled out in several phases:</p>
238<ol class="arabic simple"> 234<ol class="arabic simple">
239<li>Add support for verification of Manifest2 entries in portage</li> 235<li>Add support for verification of Manifest2 entries in portage</li>
240<li>Enable generation of Manifest2 entries in addition to the current system</li> 236<li>Enable generation of Manifest2 entries in addition to the current system</li>
241<li>Ignore digests during <tt class="docutils literal"><span class="pre">emerge</span> <span class="pre">--sync</span></tt> to get the size-benefit clientside. 237<li>Ignore digests during <tt class="docutils literal"><span class="pre">emerge</span> <span class="pre">--sync</span></tt> to get the size-benefit clientside.
242This step may be ommitted if the following steps are expected to follow soon.</li> 238This step may be ommitted if the following steps are expected to follow soon.</li>
243<li>Disable generation of entries for the current system</li> 239<li>Disable generation of entries for the current system</li>
244<li>Remove all traces of the current system from the tree (serverside)</li> 240<li>Remove all traces of the current system from the tree (serverside)</li>
245</ol> 241</ol>
246<p>Each step has its own issues. While 1) and 2) can be implemented without any 242<p>Each step has its own issues. While 1) and 2) can be implemented without any
247compability problems all later steps have a major impact:</p> 243compability problems all later steps have a major impact:</p>
248<ul class="simple"> 244<ul class="simple">
249<li>Step 3) can only be implemented when the whole tree is Manifest2 ready 245<li>Step 3) can only be implemented when the whole tree is Manifest2 ready
250(ideally speaking, practically the requirement will be more like 95% coverage 246(ideally speaking, practically the requirement will be more like 95% coverage
251with the expectation that for the remaining 5% either bugs will be filed after 247with the expectation that for the remaining 5% either bugs will be filed after
252step 3) is completed or they'll be updated at step 5).</li> 248step 3) is completed or they'll be updated at step 5).</li>
253<li>Steps 4) and 5) will render all portage versions without Manifest2 support 249<li>Steps 4) and 5) will render all portage versions without Manifest2 support
254basically useless (users would have to regenerate the digest and Manifest 250basically useless (users would have to regenerate the digest and Manifest
255for each package before being able to merge it), so this requires a almost 251for each package before being able to merge it), so this requires a almost
256100% coverage of the userbase with Manifest2 capabale portage versions 252100% coverage of the userbase with Manifest2 capabale portage versions
257(with step 1) completely implemented).</li> 253(with step 1) completely implemented).</li>
258</ul> 254</ul>
259<p>Another problem is that some steps affect different targets:</p> 255<p>Another problem is that some steps affect different targets:</p>
260<ul class="simple"> 256<ul class="simple">
261<li>Steps 1) and 3) target portage versions used by users</li> 257<li>Steps 1) and 3) target portage versions used by users</li>
262<li>Steps 2) and 4) target portage versions used by devs</li> 258<li>Steps 2) and 4) target portage versions used by devs</li>
263<li>Step 5) targets the portage tree on the cvs server</li> 259<li>Step 5) targets the portage tree on the cvs server</li>
264</ul> 260</ul>
265<p>While it is relatively easy to get all devs to use a new portage version this is 261<p>While it is relatively easy to get all devs to use a new portage version this is
266practically impossible with users as some don't update their systems regulary. 262practically impossible with users as some don't update their systems regulary.
267While six months are probably sufficient to reach a 95% coverage one year is 263While six months are probably sufficient to reach a 95% coverage one year is
268estimated to reach an almost-complete coverage. All times are relative to the 264estimated to reach an almost-complete coverage. All times are relative to the
269stable-marking of a compatible portage version.</p> 265stable-marking of a compatible portage version.</p>
270<p>No timeframe for implementation is presented here as it is highly dependent 266<p>No timeframe for implementation is presented here as it is highly dependent
271on the completion of each step.</p> 267on the completion of each step.</p>
272<p>In summary it can be said that while a full conversion will take over a year 268<p>In summary it can be said that while a full conversion will take over a year
273to be completed due to compability issues mentioned above some benefits of the 269to be completed due to compability issues mentioned above some benefits of the
274system can selectively be used as soon as step 2) is completed.</p> 270system can selectively be used as soon as step 2) is completed.</p>
275</div> 271</div>
276<div class="section" id="other-problems"> 272<div class="section">
277<h1><a class="toc-backref" href="#id21" name="other-problems">Other problems</a></h1> 273<h1><a class="toc-backref" href="#id21" id="other-problems" name="other-problems">Other problems</a></h1>
278<div class="section" id="impacts-on-infrastructure"> 274<div class="section">
279<h2><a class="toc-backref" href="#id22" name="impacts-on-infrastructure">Impacts on infrastructure</a></h2> 275<h2><a class="toc-backref" href="#id22" id="impacts-on-infrastructure" name="impacts-on-infrastructure">Impacts on infrastructure</a></h2>
280<p>While one long term goal of this proposal is to reduce the size of the tree 276<p>While one long term goal of this proposal is to reduce the size of the tree
281and therefore make life for the Gentoo Infrastructure easier this will only 277and therefore make life for the Gentoo Infrastructure easier this will only
282take effect once the implementation is rolled out completely. In the meantime 278take effect once the implementation is rolled out completely. In the meantime
283however it will increase the tree size due to keeping checksums in both formats. 279however it will increase the tree size due to keeping checksums in both formats.
284It's not possible to give a usable estimate on the degree of the increase as 280It's not possible to give a usable estimate on the degree of the increase as
285it depends on many variables such as the exact implementation timeframe, 281it depends on many variables such as the exact implementation timeframe,
286propagation of Manifest2 capable portage versions among devs or the update 282propagation of Manifest2 capable portage versions among devs or the update
287rate of the tree. It has been suggested that Manifest files that are not gpg 283rate of the tree. It has been suggested that Manifest files that are not gpg
288signed could be mass converted in one step, this could certainly help but only 284signed could be mass converted in one step, this could certainly help but only
289to some degree (according to a recent research <a class="footnote-reference" href="#gpg-numbers" id="id5" name="id5">[3]</a> about 40% of 285to some degree (according to a recent research <a class="footnote-reference" href="#gpg-numbers" id="id5" name="id5">[3]</a> about 40% of
290all Manifests in the tree are signed, but this number hasn't been verified).</p> 286all Manifests in the tree are signed, but this number hasn't been verified).</p>
291</div> 287</div>
292</div> 288</div>
293<div class="section" id="reference-implementation"> 289<div class="section">
294<h1><a class="toc-backref" href="#id23" name="reference-implementation">Reference Implementation</a></h1> 290<h1><a class="toc-backref" href="#id23" id="reference-implementation" name="reference-implementation">Reference Implementation</a></h1>
295<p>A patch for a prototype implementation of Manifest2 verification and partial 291<p>A patch for a prototype implementation of Manifest2 verification and partial
296generation has been posted at <a class="footnote-reference" href="#manifest2-patch" id="id6" name="id6">[4]</a>, it will be reworked before 292generation has been posted at <a class="footnote-reference" href="#manifest2-patch" id="id6" name="id6">[4]</a>, it will be reworked before
297being considered for inclusion in portage. However it shows that adding support 293being considered for inclusion in portage. However it shows that adding support
298for verification is quite simple, but generation is a bit tricky and will 294for verification is quite simple, but generation is a bit tricky and will
299therefore be implemented later.</p> 295therefore be implemented later.</p>
300</div> 296</div>
301<div class="section" id="options"> 297<div class="section">
302<h1><a class="toc-backref" href="#id24" name="options">Options</a></h1> 298<h1><a class="toc-backref" href="#id24" id="options" name="options">Options</a></h1>
303<p>Some things have been considered for this GLEP but aren't part of the proposal 299<p>Some things have been considered for this GLEP but aren't part of the proposal
304yet for various reasons:</p> 300yet for various reasons:</p>
305<ul class="simple"> 301<ul class="simple">
306<li>timestamp field: the author has considered adding a timestamp field for 302<li>timestamp field: the author has considered adding a timestamp field for
307each entry to list the time the entry was created. However so far no practical 303each entry to list the time the entry was created. However so far no practical
308use for such a feature has been found.</li> 304use for such a feature has been found.</li>
309<li>convert size field into checksum: Another idea was to treat the size field 305<li>convert size field into checksum: Another idea was to treat the size field
310like any other checksum. But so far no real benefit (other than a slightly 306like any other checksum. But so far no real benefit (other than a slightly
311more modular implementation) for this has been seen while it has several 307more modular implementation) for this has been seen while it has several
312drawbacks: For once, unlike checksums, the size field is definitely required 308drawbacks: For once, unlike checksums, the size field is definitely required
313for all <tt class="docutils literal"><span class="pre">DIST</span></tt> files, also it would slightly increase the length of 309for all <tt class="docutils literal"><span class="pre">DIST</span></tt> files, also it would slightly increase the length of
314each entry by adding a <tt class="docutils literal"><span class="pre">SIZE</span></tt> keyword.</li> 310each entry by adding a <tt class="docutils literal"><span class="pre">SIZE</span></tt> keyword.</li>
315<li>removal of the <tt class="docutils literal"><span class="pre">MISC</span></tt> type: It has been suggested to completely drop 311<li>removal of the <tt class="docutils literal"><span class="pre">MISC</span></tt> type: It has been suggested to completely drop
316entries of type <tt class="docutils literal"><span class="pre">MISC</span></tt>. This would result in a minor space reduction 312entries of type <tt class="docutils literal"><span class="pre">MISC</span></tt>. This would result in a minor space reduction
317(its rather unlikely to free any blocks) but completely remove the ability 313(its rather unlikely to free any blocks) but completely remove the ability
318to check these files for integrity. While they don't influence portage 314to check these files for integrity. While they don't influence portage
319or packages directly they can contain viable information for users, so 315or packages directly they can contain viable information for users, so
320the author has the opinion that at least the option for integrity checks 316the author has the opinion that at least the option for integrity checks
321should be kept.</li> 317should be kept.</li>
322</ul> 318</ul>
323</div> 319</div>
324<div class="section" id="credits"> 320<div class="section">
325<h1><a class="toc-backref" href="#id25" name="credits">Credits</a></h1> 321<h1><a class="toc-backref" href="#id25" id="credits" name="credits">Credits</a></h1>
326<p>Thanks to the following persons for their input on or related to this GLEP 322<p>Thanks to the following persons for their input on or related to this GLEP
327(even though they might not have known it): 323(even though they might not have known it):
328Ned Ludd (solar), Brian Harring (ferringb), Jason Stubbs (jstubbs), 324Ned Ludd (solar), Brian Harring (ferringb), Jason Stubbs (jstubbs),
329Robin H. Johnson (robbat2), Aron Griffis (agriffis)</p> 325Robin H. Johnson (robbat2), Aron Griffis (agriffis)</p>
330<p>Also thanks to Nicholas Jones (carpaski) to make the current Manifest system 326<p>Also thanks to Nicholas Jones (carpaski) to make the current Manifest system
331resistent enough to be able to handle this change without too many transition 327resistent enough to be able to handle this change without too many transition
332problems.</p> 328problems.</p>
333</div> 329</div>
334<div class="section" id="references"> 330<div class="section">
335<h1><a class="toc-backref" href="#id26" name="references">References</a></h1> 331<h1><a class="toc-backref" href="#id26" id="references" name="references">References</a></h1>
336<table class="docutils footnote" frame="void" id="reorg-thread" rules="none"> 332<table class="docutils footnote" frame="void" id="reorg-thread" rules="none">
337<colgroup><col class="label" /><col /></colgroup> 333<colgroup><col class="label" /><col /></colgroup>
338<tbody valign="top"> 334<tbody valign="top">
339<tr><td class="label"><a class="fn-backref" href="#id1" name="reorg-thread">[1]</a></td><td><a class="reference" href="http://thread.gmane.org/gmane.linux.gentoo.devel/21920">http://thread.gmane.org/gmane.linux.gentoo.devel/21920</a></td></tr> 335<tr><td class="label"><a class="fn-backref" href="#id1" name="reorg-thread">[1]</a></td><td><a class="reference" href="http://thread.gmane.org/gmane.linux.gentoo.devel/21920">http://thread.gmane.org/gmane.linux.gentoo.devel/21920</a></td></tr>
340</tbody> 336</tbody>
365</tbody> 361</tbody>
366</table> 362</table>
367<table class="docutils footnote" frame="void" id="id7" rules="none"> 363<table class="docutils footnote" frame="void" id="id7" rules="none">
368<colgroup><col class="label" /><col /></colgroup> 364<colgroup><col class="label" /><col /></colgroup>
369<tbody valign="top"> 365<tbody valign="top">
370<tr><td class="label"><a class="fn-backref" href="#id8" name="id7">[6]</a></td><td><a class="reference" href="glep-0044-extras/manifest2-example">glep-0044-extras/manifest2-example</a></td></tr> 366<tr><td class="label"><a class="fn-backref" href="#id8" name="id7">[6]</a></td><td><a class="reference" href="glep-0044-extras/manifest2-example.txt">glep-0044-extras/manifest2-example.txt</a></td></tr>
371</tbody> 367</tbody>
372</table> 368</table>
373</div> 369</div>
374<div class="section" id="copyright"> 370<div class="section">
375<h1><a class="toc-backref" href="#id27" name="copyright">Copyright</a></h1> 371<h1><a class="toc-backref" href="#id27" id="copyright" name="copyright">Copyright</a></h1>
376<p>This document has been placed in the public domain.</p> 372<p>This document has been placed in the public domain.</p>
377</div> 373</div>
378 374
379</div> 375</div>
380<div class="footer"> 376<div class="footer">
381<hr class="footer" /> 377<hr class="footer" />
382<a class="reference" href="glep-0044.txt">View document source</a>. 378<a class="reference" href="glep-0044.txt">View document source</a>.
383Generated on: 2006-02-10 23:25 UTC. 379Generated on: 2007-10-13 13:39 UTC.
384Generated by <a class="reference" href="http://docutils.sourceforge.net/">Docutils</a> from <a class="reference" href="http://docutils.sourceforge.net/rst.html">reStructuredText</a> source. 380Generated by <a class="reference" href="http://docutils.sourceforge.net/">Docutils</a> from <a class="reference" href="http://docutils.sourceforge.net/rst.html">reStructuredText</a> source.
385 381
386</div> 382</div>
387</body> 383</body>
388</html> 384</html>

Legend:
Removed from v.1.4  
changed lines
  Added in v.1.10

  ViewVC Help
Powered by ViewVC 1.1.20