| 1 |
g2boojum |
1.1 |
<?xml version="1.0" encoding="utf-8" ?>
|
| 2 |
|
|
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
|
| 3 |
|
|
<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
|
| 4 |
|
|
<!--
|
| 5 |
|
|
This HTML is auto-generated. DO NOT EDIT THIS FILE! If you are writing a new
|
| 6 |
|
|
PEP, see http://www.python.org/peps/pep-0001.html for instructions and links
|
| 7 |
|
|
to templates. DO NOT USE THIS HTML FILE AS YOUR TEMPLATE!
|
| 8 |
|
|
-->
|
| 9 |
|
|
<head>
|
| 10 |
|
|
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
| 11 |
g2boojum |
1.4 |
<meta name="generator" content="Docutils 0.4: http://docutils.sourceforge.net/" />
|
| 12 |
g2boojum |
1.1 |
<title>GLEP 25 -- Distfile Patching Support</title>
|
| 13 |
g2boojum |
1.4 |
<style type="text/css">
|
| 14 |
|
|
|
| 15 |
|
|
/*
|
| 16 |
|
|
:Author: David Goodger
|
| 17 |
|
|
:Contact: goodger@users.sourceforge.net
|
| 18 |
|
|
:date: $Date: 2003/06/02 17:03:08 $
|
| 19 |
|
|
:version: $Revision: 1.1 $
|
| 20 |
|
|
:copyright: This stylesheet has been placed in the public domain.
|
| 21 |
|
|
|
| 22 |
|
|
Default cascading style sheet for the PEP HTML output of Docutils.
|
| 23 |
|
|
*/
|
| 24 |
|
|
|
| 25 |
|
|
.first {
|
| 26 |
|
|
margin-top: 0 }
|
| 27 |
|
|
|
| 28 |
|
|
.last {
|
| 29 |
|
|
margin-bottom: 0 }
|
| 30 |
|
|
|
| 31 |
|
|
.navigation {
|
| 32 |
|
|
width: 100% ;
|
| 33 |
|
|
background: #cc99ff ;
|
| 34 |
|
|
margin-top: 0px ;
|
| 35 |
|
|
margin-bottom: 0px }
|
| 36 |
|
|
|
| 37 |
|
|
.navigation .navicon {
|
| 38 |
|
|
width: 150px ;
|
| 39 |
|
|
height: 35px }
|
| 40 |
|
|
|
| 41 |
|
|
.navigation .textlinks {
|
| 42 |
|
|
padding-left: 1em ;
|
| 43 |
|
|
text-align: left }
|
| 44 |
|
|
|
| 45 |
|
|
.navigation td, .navigation th {
|
| 46 |
|
|
padding-left: 0em ;
|
| 47 |
|
|
padding-right: 0em ;
|
| 48 |
|
|
vertical-align: middle }
|
| 49 |
|
|
|
| 50 |
|
|
.rfc2822 {
|
| 51 |
|
|
margin-top: 0.5em ;
|
| 52 |
|
|
margin-left: 0.5em ;
|
| 53 |
|
|
margin-right: 0.5em ;
|
| 54 |
|
|
margin-bottom: 0em }
|
| 55 |
|
|
|
| 56 |
|
|
.rfc2822 td {
|
| 57 |
|
|
text-align: left }
|
| 58 |
|
|
|
| 59 |
|
|
.rfc2822 th.field-name {
|
| 60 |
|
|
text-align: right ;
|
| 61 |
|
|
font-family: sans-serif ;
|
| 62 |
|
|
padding-right: 0.5em ;
|
| 63 |
|
|
font-weight: bold ;
|
| 64 |
|
|
margin-bottom: 0em }
|
| 65 |
|
|
|
| 66 |
|
|
a.toc-backref {
|
| 67 |
|
|
text-decoration: none ;
|
| 68 |
|
|
color: black }
|
| 69 |
|
|
|
| 70 |
|
|
body {
|
| 71 |
|
|
margin: 0px ;
|
| 72 |
|
|
margin-bottom: 1em ;
|
| 73 |
|
|
padding: 0px }
|
| 74 |
|
|
|
| 75 |
|
|
dd {
|
| 76 |
|
|
margin-bottom: 0.5em }
|
| 77 |
|
|
|
| 78 |
|
|
div.section {
|
| 79 |
|
|
margin-left: 1em ;
|
| 80 |
|
|
margin-right: 1em ;
|
| 81 |
|
|
margin-bottom: 1.5em }
|
| 82 |
|
|
|
| 83 |
|
|
div.section div.section {
|
| 84 |
|
|
margin-left: 0em ;
|
| 85 |
|
|
margin-right: 0em ;
|
| 86 |
|
|
margin-top: 1.5em }
|
| 87 |
|
|
|
| 88 |
|
|
div.abstract {
|
| 89 |
|
|
margin: 2em 5em }
|
| 90 |
|
|
|
| 91 |
|
|
div.abstract p.topic-title {
|
| 92 |
|
|
font-weight: bold ;
|
| 93 |
|
|
text-align: center }
|
| 94 |
|
|
|
| 95 |
|
|
div.attention, div.caution, div.danger, div.error, div.hint,
|
| 96 |
|
|
div.important, div.note, div.tip, div.warning {
|
| 97 |
|
|
margin: 2em ;
|
| 98 |
|
|
border: medium outset ;
|
| 99 |
|
|
padding: 1em }
|
| 100 |
|
|
|
| 101 |
|
|
div.attention p.admonition-title, div.caution p.admonition-title,
|
| 102 |
|
|
div.danger p.admonition-title, div.error p.admonition-title,
|
| 103 |
|
|
div.warning p.admonition-title {
|
| 104 |
|
|
color: red ;
|
| 105 |
|
|
font-weight: bold ;
|
| 106 |
|
|
font-family: sans-serif }
|
| 107 |
|
|
|
| 108 |
|
|
div.hint p.admonition-title, div.important p.admonition-title,
|
| 109 |
|
|
div.note p.admonition-title, div.tip p.admonition-title {
|
| 110 |
|
|
font-weight: bold ;
|
| 111 |
|
|
font-family: sans-serif }
|
| 112 |
|
|
|
| 113 |
|
|
div.figure {
|
| 114 |
|
|
margin-left: 2em }
|
| 115 |
|
|
|
| 116 |
|
|
div.footer, div.header {
|
| 117 |
|
|
font-size: smaller }
|
| 118 |
|
|
|
| 119 |
|
|
div.footer {
|
| 120 |
|
|
margin-left: 1em ;
|
| 121 |
|
|
margin-right: 1em }
|
| 122 |
|
|
|
| 123 |
|
|
div.system-messages {
|
| 124 |
|
|
margin: 5em }
|
| 125 |
|
|
|
| 126 |
|
|
div.system-messages h1 {
|
| 127 |
|
|
color: red }
|
| 128 |
|
|
|
| 129 |
|
|
div.system-message {
|
| 130 |
|
|
border: medium outset ;
|
| 131 |
|
|
padding: 1em }
|
| 132 |
|
|
|
| 133 |
|
|
div.system-message p.system-message-title {
|
| 134 |
|
|
color: red ;
|
| 135 |
|
|
font-weight: bold }
|
| 136 |
|
|
|
| 137 |
|
|
div.topic {
|
| 138 |
|
|
margin: 2em }
|
| 139 |
|
|
|
| 140 |
|
|
h1 {
|
| 141 |
|
|
font-family: sans-serif ;
|
| 142 |
|
|
font-size: large }
|
| 143 |
|
|
|
| 144 |
|
|
h2 {
|
| 145 |
|
|
font-family: sans-serif ;
|
| 146 |
|
|
font-size: medium }
|
| 147 |
|
|
|
| 148 |
|
|
h3 {
|
| 149 |
|
|
font-family: sans-serif ;
|
| 150 |
|
|
font-size: small }
|
| 151 |
|
|
|
| 152 |
|
|
h4 {
|
| 153 |
|
|
font-family: sans-serif ;
|
| 154 |
|
|
font-style: italic ;
|
| 155 |
|
|
font-size: small }
|
| 156 |
|
|
|
| 157 |
|
|
h5 {
|
| 158 |
|
|
font-family: sans-serif;
|
| 159 |
|
|
font-size: x-small }
|
| 160 |
|
|
|
| 161 |
|
|
h6 {
|
| 162 |
|
|
font-family: sans-serif;
|
| 163 |
|
|
font-style: italic ;
|
| 164 |
|
|
font-size: x-small }
|
| 165 |
|
|
|
| 166 |
|
|
.section hr {
|
| 167 |
|
|
width: 75% }
|
| 168 |
|
|
|
| 169 |
|
|
ol.simple, ul.simple {
|
| 170 |
|
|
margin-bottom: 1em }
|
| 171 |
|
|
|
| 172 |
|
|
ol.arabic {
|
| 173 |
|
|
list-style: decimal }
|
| 174 |
|
|
|
| 175 |
|
|
ol.loweralpha {
|
| 176 |
|
|
list-style: lower-alpha }
|
| 177 |
|
|
|
| 178 |
|
|
ol.upperalpha {
|
| 179 |
|
|
list-style: upper-alpha }
|
| 180 |
|
|
|
| 181 |
|
|
ol.lowerroman {
|
| 182 |
|
|
list-style: lower-roman }
|
| 183 |
|
|
|
| 184 |
|
|
ol.upperroman {
|
| 185 |
|
|
list-style: upper-roman }
|
| 186 |
|
|
|
| 187 |
|
|
p.caption {
|
| 188 |
|
|
font-style: italic }
|
| 189 |
|
|
|
| 190 |
|
|
p.credits {
|
| 191 |
|
|
font-style: italic ;
|
| 192 |
|
|
font-size: smaller }
|
| 193 |
|
|
|
| 194 |
|
|
p.label {
|
| 195 |
|
|
white-space: nowrap }
|
| 196 |
|
|
|
| 197 |
|
|
p.topic-title {
|
| 198 |
|
|
font-family: sans-serif ;
|
| 199 |
|
|
font-weight: bold }
|
| 200 |
|
|
|
| 201 |
|
|
pre.line-block {
|
| 202 |
|
|
font-family: serif ;
|
| 203 |
|
|
font-size: 100% }
|
| 204 |
|
|
|
| 205 |
|
|
pre.literal-block, pre.doctest-block {
|
| 206 |
|
|
margin-left: 2em ;
|
| 207 |
|
|
margin-right: 2em ;
|
| 208 |
|
|
background-color: #eeeeee }
|
| 209 |
|
|
|
| 210 |
|
|
span.classifier {
|
| 211 |
|
|
font-family: sans-serif ;
|
| 212 |
|
|
font-style: oblique }
|
| 213 |
|
|
|
| 214 |
|
|
span.classifier-delimiter {
|
| 215 |
|
|
font-family: sans-serif ;
|
| 216 |
|
|
font-weight: bold }
|
| 217 |
|
|
|
| 218 |
|
|
span.interpreted {
|
| 219 |
|
|
font-family: sans-serif }
|
| 220 |
|
|
|
| 221 |
|
|
span.option-argument {
|
| 222 |
|
|
font-style: italic }
|
| 223 |
|
|
|
| 224 |
|
|
span.pre {
|
| 225 |
|
|
white-space: pre }
|
| 226 |
|
|
|
| 227 |
|
|
span.problematic {
|
| 228 |
|
|
color: red }
|
| 229 |
|
|
|
| 230 |
|
|
table {
|
| 231 |
|
|
margin-top: 0.5em ;
|
| 232 |
|
|
margin-bottom: 0.5em }
|
| 233 |
|
|
|
| 234 |
|
|
td, th {
|
| 235 |
|
|
padding-left: 0.5em ;
|
| 236 |
|
|
padding-right: 0.5em ;
|
| 237 |
|
|
vertical-align: top }
|
| 238 |
|
|
|
| 239 |
|
|
td.num {
|
| 240 |
|
|
text-align: right }
|
| 241 |
|
|
|
| 242 |
|
|
th.field-name {
|
| 243 |
|
|
font-weight: bold ;
|
| 244 |
|
|
text-align: left ;
|
| 245 |
|
|
white-space: nowrap }
|
| 246 |
|
|
|
| 247 |
|
|
h1 tt, h2 tt, h3 tt, h4 tt, h5 tt, h6 tt {
|
| 248 |
|
|
font-size: 100% }
|
| 249 |
|
|
|
| 250 |
|
|
tt {
|
| 251 |
|
|
background-color: #eeeeee }
|
| 252 |
|
|
|
| 253 |
|
|
ul.auto-toc {
|
| 254 |
|
|
list-style-type: none }
|
| 255 |
|
|
|
| 256 |
|
|
</style>
|
| 257 |
g2boojum |
1.1 |
</head>
|
| 258 |
|
|
<body bgcolor="white">
|
| 259 |
|
|
<table class="navigation" cellpadding="0" cellspacing="0"
|
| 260 |
|
|
width="100%" border="0">
|
| 261 |
|
|
<tr><td class="navicon" width="150" height="35">
|
| 262 |
|
|
<a href="http://www.gentoo.org/" title="Gentoo Linux Home Page">
|
| 263 |
|
|
<img src="http://www.gentoo.org/images/gentoo-new.gif" alt="[Gentoo]"
|
| 264 |
|
|
border="0" width="150" height="35" /></a></td>
|
| 265 |
|
|
<td class="textlinks" align="left">
|
| 266 |
|
|
[<b><a href="http://www.gentoo.org/">Gentoo Linux Home</a></b>]
|
| 267 |
g2boojum |
1.4 |
[<b><a href="http://www.gentoo.org/peps">GLEP Index</a></b>]
|
| 268 |
|
|
[<b><a href="http://www.gentoo.org/proj/en/glep/glep-0025.txt">GLEP Source</a></b>]
|
| 269 |
g2boojum |
1.1 |
</td></tr></table>
|
| 270 |
g2boojum |
1.3 |
<table class="rfc2822 docutils field-list" frame="void" rules="none">
|
| 271 |
g2boojum |
1.1 |
<col class="field-name" />
|
| 272 |
|
|
<col class="field-body" />
|
| 273 |
|
|
<tbody valign="top">
|
| 274 |
|
|
<tr class="field"><th class="field-name">GLEP:</th><td class="field-body">25</td>
|
| 275 |
|
|
</tr>
|
| 276 |
|
|
<tr class="field"><th class="field-name">Title:</th><td class="field-body">Distfile Patching Support</td>
|
| 277 |
|
|
</tr>
|
| 278 |
g2boojum |
1.3 |
<tr class="field"><th class="field-name">Version:</th><td class="field-body">1.3</td>
|
| 279 |
g2boojum |
1.1 |
</tr>
|
| 280 |
g2boojum |
1.4 |
<tr class="field"><th class="field-name">Last-Modified:</th><td class="field-body"><a class="reference" href="http://www.gentoo.org/cgi-bin/viewcvs.cgi/xml/htdocs/proj/en/glep/glep-0025.txt?cvsroot=gentoo">2005/04/01 01:32:19</a></td>
|
| 281 |
g2boojum |
1.1 |
</tr>
|
| 282 |
|
|
<tr class="field"><th class="field-name">Author:</th><td class="field-body">Brian Harring <ferringb at gentoo.org></td>
|
| 283 |
|
|
</tr>
|
| 284 |
g2boojum |
1.2 |
<tr class="field"><th class="field-name">Status:</th><td class="field-body">deferred</td>
|
| 285 |
g2boojum |
1.1 |
</tr>
|
| 286 |
|
|
<tr class="field"><th class="field-name">Type:</th><td class="field-body">Standards Track</td>
|
| 287 |
|
|
</tr>
|
| 288 |
g2boojum |
1.4 |
<tr class="field"><th class="field-name">Content-Type:</th><td class="field-body"><a class="reference" href="glep-0002.html">text/x-rst</a></td>
|
| 289 |
g2boojum |
1.1 |
</tr>
|
| 290 |
|
|
<tr class="field"><th class="field-name">Created:</th><td class="field-body">6-Mar-2004</td>
|
| 291 |
|
|
</tr>
|
| 292 |
g2boojum |
1.2 |
<tr class="field"><th class="field-name">Post-History:</th><td class="field-body">4-Apr-2004, 11-Nov-2004</td>
|
| 293 |
g2boojum |
1.1 |
</tr>
|
| 294 |
|
|
</tbody>
|
| 295 |
|
|
</table>
|
| 296 |
|
|
<hr />
|
| 297 |
g2boojum |
1.4 |
<div class="contents topic">
|
| 298 |
|
|
<p class="topic-title first"><a id="contents" name="contents">Contents</a></p>
|
| 299 |
g2boojum |
1.1 |
<ul class="simple">
|
| 300 |
|
|
<li><a class="reference" href="#abstract" id="id7" name="id7">Abstract</a></li>
|
| 301 |
g2boojum |
1.2 |
<li><a class="reference" href="#status" id="id8" name="id8">Status</a></li>
|
| 302 |
|
|
<li><a class="reference" href="#motivation" id="id9" name="id9">Motivation</a></li>
|
| 303 |
|
|
<li><a class="reference" href="#binary-patches-vs-gnudiff-patches" id="id10" name="id10">Binary patches vs GNUDiff patches</a></li>
|
| 304 |
|
|
<li><a class="reference" href="#rationale" id="id11" name="id11">Rationale</a></li>
|
| 305 |
|
|
<li><a class="reference" href="#specification" id="id12" name="id12">Specification</a><ul>
|
| 306 |
|
|
<li><a class="reference" href="#additions-to-the-tree" id="id13" name="id13">Additions to the tree</a></li>
|
| 307 |
|
|
<li><a class="reference" href="#portage-implementation" id="id14" name="id14">Portage Implementation</a><ul>
|
| 308 |
|
|
<li><a class="reference" href="#fetching" id="id15" name="id15">Fetching</a></li>
|
| 309 |
|
|
<li><a class="reference" href="#reconstruction" id="id16" name="id16">Reconstruction</a></li>
|
| 310 |
|
|
<li><a class="reference" href="#compressed-md5sums" id="id17" name="id17">Compressed MD5sums:</a><ul>
|
| 311 |
|
|
<li><a class="reference" href="#the-problem-in-detail" id="id18" name="id18">The Problem in Detail</a></li>
|
| 312 |
|
|
<li><a class="reference" href="#the-proposed-solution" id="id19" name="id19">The Proposed Solution</a></li>
|
| 313 |
g2boojum |
1.1 |
</ul>
|
| 314 |
|
|
</li>
|
| 315 |
|
|
</ul>
|
| 316 |
|
|
</li>
|
| 317 |
g2boojum |
1.2 |
<li><a class="reference" href="#distfile-mirror-additions" id="id20" name="id20">Distfile Mirror Additions</a></li>
|
| 318 |
|
|
<li><a class="reference" href="#patch-creation" id="id21" name="id21">Patch Creation</a></li>
|
| 319 |
g2boojum |
1.1 |
</ul>
|
| 320 |
|
|
</li>
|
| 321 |
g2boojum |
1.3 |
<li><a class="reference" href="#backwards-compatibility" id="id22" name="id22">Backwards Compatibility</a></li>
|
| 322 |
g2boojum |
1.2 |
<li><a class="reference" href="#reference-implementation" id="id23" name="id23">Reference Implementation</a></li>
|
| 323 |
|
|
<li><a class="reference" href="#references" id="id24" name="id24">References</a></li>
|
| 324 |
|
|
<li><a class="reference" href="#copyright" id="id25" name="id25">Copyright</a></li>
|
| 325 |
g2boojum |
1.1 |
</ul>
|
| 326 |
|
|
</div>
|
| 327 |
g2boojum |
1.4 |
<div class="section">
|
| 328 |
|
|
<h1><a class="toc-backref" href="#id7" id="abstract" name="abstract">Abstract</a></h1>
|
| 329 |
g2boojum |
1.1 |
<p>The intention of this GLEP is to propose the creation of patching support for
|
| 330 |
|
|
portage, and iron out the implementation details.</p>
|
| 331 |
|
|
</div>
|
| 332 |
g2boojum |
1.4 |
<div class="section">
|
| 333 |
|
|
<h1><a class="toc-backref" href="#id8" id="status" name="status">Status</a></h1>
|
| 334 |
g2boojum |
1.2 |
<p>Timed out</p>
|
| 335 |
|
|
</div>
|
| 336 |
g2boojum |
1.4 |
<div class="section">
|
| 337 |
|
|
<h1><a class="toc-backref" href="#id9" id="motivation" name="motivation">Motivation</a></h1>
|
| 338 |
g2boojum |
1.1 |
<p>Reduce the bandwidth load placed on our mirrors by decreasing the amount of
|
| 339 |
|
|
bytes transferred when upgrading between versions. Side benefit of this is to
|
| 340 |
|
|
significantly decrease the download requirements for users lacking broadband.</p>
|
| 341 |
|
|
</div>
|
| 342 |
g2boojum |
1.4 |
<div class="section">
|
| 343 |
|
|
<h1><a class="toc-backref" href="#id10" id="binary-patches-vs-gnudiff-patches" name="binary-patches-vs-gnudiff-patches">Binary patches vs GNUDiff patches</a></h1>
|
| 344 |
g2boojum |
1.1 |
<p>Most people are familiar with diff patches (unified diff for example)- this
|
| 345 |
|
|
glep is specifically proposing the use of an actual binary differencer. The
|
| 346 |
|
|
reason for this is that diff patches are line based- you change a single
|
| 347 |
|
|
character in a line, and the whole line must be included in the patch. Binary
|
| 348 |
|
|
differencers work at the byte level- it encodes just that byte. In that
|
| 349 |
|
|
respect binary patches are often much more efficient then diff patches.</p>
|
| 350 |
|
|
<p>Further, the ability to reverse a unified patch is due to the fact the diff
|
| 351 |
|
|
includes <strong>both</strong> the original line, and the modified line. The author isn't
|
| 352 |
|
|
aware of any binary differencer that is able to create patches the can be
|
| 353 |
|
|
reversed- basically they're unidirectional, the patch that is generated can
|
| 354 |
|
|
only be used to upgrade or downgrade the version, not both. The plus side of
|
| 355 |
|
|
this limitation is a significantly decreased patch size.</p>
|
| 356 |
|
|
<p>The choice of binary patches over diff patches pretty much comes down to the
|
| 357 |
|
|
fact they're smaller- example being a kdelibs binary patch for 3.1.4->3.1.5 is
|
| 358 |
|
|
75kb, the equivalent diff patch is 123kb, and is unable to result in a correct
|
| 359 |
|
|
md5 <a class="footnote-reference" href="#id4" id="id1" name="id1">[1]</a>.</p>
|
| 360 |
|
|
<p>Currently, this glep is proposing only the usage of binary patches- that's not
|
| 361 |
|
|
to say (with a fair amount of work) it couldn't be extended to support
|
| 362 |
|
|
standard diffs.</p>
|
| 363 |
|
|
</div>
|
| 364 |
g2boojum |
1.4 |
<div class="section">
|
| 365 |
|
|
<h1><a class="toc-backref" href="#id11" id="rationale" name="rationale">Rationale</a></h1>
|
| 366 |
g2boojum |
1.1 |
<p>The difference between source releases typically isn't very large, especially
|
| 367 |
|
|
for minor releases. As an example, kdelibs-3.1.4.tar.bz2 is 10.53 MB, and
|
| 368 |
|
|
kdelibs-3.1.5.tar.bz2 is 10.54 MB. A bzip2'ed patch between those versions is
|
| 369 |
|
|
75.6 kb <a class="footnote-reference" href="#id5" id="id2" name="id2">[2]</a>, less then 1% the size of 3.1.5's tbz2.</p>
|
| 370 |
|
|
</div>
|
| 371 |
g2boojum |
1.4 |
<div class="section">
|
| 372 |
|
|
<h1><a class="toc-backref" href="#id12" id="specification" name="specification">Specification</a></h1>
|
| 373 |
g2boojum |
1.1 |
<p>Quite a few sections of gentoo are affected- mirroring, the portage tree, and
|
| 374 |
|
|
portage itself.</p>
|
| 375 |
g2boojum |
1.4 |
<div class="section">
|
| 376 |
|
|
<h2><a class="toc-backref" href="#id13" id="additions-to-the-tree" name="additions-to-the-tree">Additions to the tree</a></h2>
|
| 377 |
g2boojum |
1.1 |
<p>For adding patch info into the tree, this glep proposes a global patch list
|
| 378 |
|
|
(stored in profiles as patches.global), and individual patch lists stored in
|
| 379 |
|
|
relevant package directories (named patches). Using the kernel packages as an
|
| 380 |
|
|
example, a global list of patches enables us to create a patch once, add an
|
| 381 |
|
|
entry, and have all kernel packages benefit from that single entry. Both
|
| 382 |
|
|
patches.global, and individual package patch files share the same format:</p>
|
| 383 |
|
|
<pre class="literal-block">
|
| 384 |
|
|
MD5 md5-value patch-url size MD5 md5-value ref-file size UMD5 md5-value new-file size
|
| 385 |
|
|
</pre>
|
| 386 |
|
|
<p>For those familiar with digest file layout, this should look familiar.
|
| 387 |
|
|
Essentially, chksum type, value, filename, size. The UMD5 chksum type is just
|
| 388 |
|
|
the uncompressed md5/size of the file- so if the UMD5 were for a bzip2
|
| 389 |
|
|
compressed file, it would be the md5 value/size of the uncompressed file.
|
| 390 |
|
|
And an example:</p>
|
| 391 |
|
|
<pre class="literal-block">
|
| 392 |
|
|
MD5 ccd5411b3558326cbce0306fcae32e26 http://dev.gentoo.org/~ferringb/patches/kdelibs-3.1.4-3.1.5.patch.bz2 75687 MD5 82c265de78d53c7060a09c5cb1a78942 kdelibs-3.1.4.tar.bz2 10537433 UMD5 0b1908a51e739c07ff5a88e189d2f7a9 kdelibs-3.1.5.tar.bz2 48056320
|
| 393 |
|
|
</pre>
|
| 394 |
|
|
<p>In the above example, the md5sum of
|
| 395 |
|
|
<a class="reference" href="http://dev.gentoo.org/~ferringb/patches/kdelibs-3.1.4-3.1.5.patch.bz2">http://dev.gentoo.org/~ferringb/patches/kdelibs-3.1.4-3.1.5.patch.bz2</a> is
|
| 396 |
|
|
calculated, compared to the stored value, and then the file size is checked.
|
| 397 |
|
|
The one difference is the UMD5 checksum type- the md5 value and the size are
|
| 398 |
|
|
specific to the <em>uncompressed</em> file. Continuing, for cases where the patch
|
| 399 |
|
|
will reside on one of our mirrors, the patch filename would be sufficient.</p>
|
| 400 |
|
|
<p>Finally, note that this is a unidirectional patch- using the above example,
|
| 401 |
|
|
kdelibs-3.1.4-3.1.5 can <strong>only</strong> be used to upgrade from 3.1.4 to 3.1.5, not
|
| 402 |
|
|
in reverse (originally explained in <a class="reference" href="#binary-patches-vs-gnudiff-patches">Binary patches vs GNUDiff patches</a>).</p>
|
| 403 |
|
|
</div>
|
| 404 |
g2boojum |
1.4 |
<div class="section">
|
| 405 |
|
|
<h2><a class="toc-backref" href="#id14" id="portage-implementation" name="portage-implementation">Portage Implementation</a></h2>
|
| 406 |
g2boojum |
1.1 |
<p>This glep proposes the patching support should be (at this stage) optional-
|
| 407 |
|
|
specifically, enabled via FEATURES="patching".</p>
|
| 408 |
g2boojum |
1.4 |
<div class="section">
|
| 409 |
|
|
<h3><a class="toc-backref" href="#id15" id="fetching" name="fetching">Fetching</a></h3>
|
| 410 |
g2boojum |
1.1 |
<p>When patching is enabled, the global patch list is read, and the packages
|
| 411 |
|
|
patch list is read. From there, portage determines what files could be used
|
| 412 |
|
|
as a base for patching to the desired file- further, determining if it's
|
| 413 |
|
|
actually worth patching (case where it wouldn't be is when the target file is
|
| 414 |
|
|
less then the sum of the patches needed). Any patches to be used are fetched,
|
| 415 |
|
|
and md5 verified.</p>
|
| 416 |
|
|
</div>
|
| 417 |
g2boojum |
1.4 |
<div class="section">
|
| 418 |
|
|
<h3><a class="toc-backref" href="#id16" id="reconstruction" name="reconstruction">Reconstruction</a></h3>
|
| 419 |
g2boojum |
1.1 |
<p>Upon fetching and md5 verification of patch(es), the desired file is
|
| 420 |
|
|
reconstructed. Assuming reconstruction didn't return any errors, the target
|
| 421 |
|
|
file has its uncompressed md5sum calculated and verified, then is recompressed
|
| 422 |
|
|
and the compressed md5sum calculated. At this point, if the compressed md5
|
| 423 |
|
|
matches the md5 stored in the tree, then portage transfers the file into
|
| 424 |
|
|
distfiles, and continues on it's merry way.</p>
|
| 425 |
|
|
<p>If the compressed md5 is different from the tree's value, then the (proposed)
|
| 426 |
|
|
md5 database is updated with new compressed md5. Details of this database
|
| 427 |
|
|
(and the issue it addresses) follow.</p>
|
| 428 |
|
|
</div>
|
| 429 |
g2boojum |
1.4 |
<div class="section">
|
| 430 |
|
|
<h3><a class="toc-backref" href="#id17" id="compressed-md5sums" name="compressed-md5sums">Compressed MD5sums:</a></h3>
|
| 431 |
g2boojum |
1.1 |
<p>There will be instances where a file is reconstructed perfectly, recompressed,
|
| 432 |
|
|
and the recompressed md5sum differs from what is stored in the tree- the
|
| 433 |
|
|
problem is that the md5sum of a compressed file is inherently tied to the
|
| 434 |
|
|
compressor version/options used to compress the original source.</p>
|
| 435 |
g2boojum |
1.4 |
<div class="section">
|
| 436 |
|
|
<h4><a class="toc-backref" href="#id18" id="the-problem-in-detail" name="the-problem-in-detail">The Problem in Detail</a></h4>
|
| 437 |
g2boojum |
1.1 |
<p>A good example of this problem is related to bzip2 versions used for
|
| 438 |
|
|
compression. Between bzip2 0.9x and bzip2 1.x, there was a subtle change in
|
| 439 |
|
|
the compressor resulting in a slightly better compression result- end result
|
| 440 |
|
|
being a different file, eg a different md5sum. Assuming compressor versions
|
| 441 |
|
|
are the same, there also is the issue of what compression level the target
|
| 442 |
|
|
source was originally compressed at- was it compressed with -9, -8 or -7?
|
| 443 |
|
|
That's just a sampling of the various original settings that must be accounted
|
| 444 |
|
|
for, and that's limited to gzip/bzip2; other compressors will add to the
|
| 445 |
|
|
number of variables to be accounted for to produce an exact recreation of the
|
| 446 |
|
|
compressed md5sum.</p>
|
| 447 |
|
|
<p>Tracking the compressor version and options originally used isn't really a
|
| 448 |
|
|
valid option- assuming all options were accounted for, clients would still be
|
| 449 |
|
|
required to have multiple versions of the same compressor installed just for
|
| 450 |
|
|
the sake of recreating a compressed md5sum <em>even though</em> the uncompressed
|
| 451 |
|
|
source's md5 has already been verified.</p>
|
| 452 |
|
|
</div>
|
| 453 |
g2boojum |
1.4 |
<div class="section">
|
| 454 |
|
|
<h4><a class="toc-backref" href="#id19" id="the-proposed-solution" name="the-proposed-solution">The Proposed Solution</a></h4>
|
| 455 |
g2boojum |
1.1 |
<p>The creation of a clientside flatfile/db of valid alternate md5/size pairs
|
| 456 |
|
|
would enable portage to handle perfectly reconstructed files, that have a
|
| 457 |
|
|
different md5sum due to compression differences. The proposed format is thus:</p>
|
| 458 |
|
|
<pre class="literal-block">
|
| 459 |
|
|
MD5 md5sum orig-file size MD5 md5sum [ optional new-name ] size
|
| 460 |
|
|
</pre>
|
| 461 |
|
|
<p>Example:</p>
|
| 462 |
|
|
<pre class="literal-block">
|
| 463 |
|
|
MD5 984146931906a7d53300b29f58f6a899 OOo_1.0.3_source.tar.bz2 165475319 MD5 0733dd85ed44d88d1eabed704d579721 165444187
|
| 464 |
|
|
</pre>
|
| 465 |
|
|
<p>An alternate md5/size pair for a file would be added <strong>only</strong> when the
|
| 466 |
|
|
uncompressed source's md5/size has been verified, yet upon recompression the
|
| 467 |
|
|
md5 differs. For cleansing of older md5/size pairs from this db, a utility
|
| 468 |
|
|
would be required- the author suggests the addition of a distfiles-cleaning
|
| 469 |
|
|
utility to portage, with the ability to also cleanse old md5/size pairs when
|
| 470 |
|
|
the file the pair was created for no longer exists in distfiles.</p>
|
| 471 |
|
|
<p>Where to store the database is debatable- /etc/portage or /var/cache/edb are
|
| 472 |
|
|
definite options.</p>
|
| 473 |
|
|
<p>The reasoning for allowing for an optional new-name is that it provides needed
|
| 474 |
|
|
functionality should anyone attempt to extend portage to allow for clients to
|
| 475 |
|
|
change the compression used for a source (eg, recompress all gzip files as
|
| 476 |
|
|
bzip2). Granted, no such code or attempt has been made, but nothing is lost
|
| 477 |
|
|
by leaving the option open should the request/attempt be made.</p>
|
| 478 |
|
|
<p>A potential gotcha of adding this support is that in environments where the
|
| 479 |
|
|
distfiles directory is shared out to multiple systems, this db must be shared
|
| 480 |
|
|
also.</p>
|
| 481 |
|
|
</div>
|
| 482 |
|
|
</div>
|
| 483 |
|
|
</div>
|
| 484 |
g2boojum |
1.4 |
<div class="section">
|
| 485 |
|
|
<h2><a class="toc-backref" href="#id20" id="distfile-mirror-additions" name="distfile-mirror-additions">Distfile Mirror Additions</a></h2>
|
| 486 |
g2boojum |
1.1 |
<p>One issue of contention is where these files will actually be stored. As of
|
| 487 |
|
|
the writing of this glep, a full distfiles mirror is roughly around 40 gb- a
|
| 488 |
|
|
rough estimate by the author places the space requirements for patches for
|
| 489 |
|
|
each version at a total of around 4gb. Note this isn't even remotely a hard
|
| 490 |
|
|
figure yet, and a better figure is being checked into currently.</p>
|
| 491 |
|
|
<p>Regardless of the exact space figure, finding a place to store the patches
|
| 492 |
|
|
will be problematic. Expansion of the required mirror space (essentially just
|
| 493 |
|
|
swallowing the patches storage requirement) is unlikely, since it was one of
|
| 494 |
|
|
the main arguements against the now defunct glep9 attempt <a class="footnote-reference" href="#id5" id="id3" name="id3">[2]</a>. A couple of
|
| 495 |
|
|
ideas that have been put forth to handle the additional space requirements are
|
| 496 |
|
|
as follows-</p>
|
| 497 |
|
|
<p>1) Identification of mirrors willing to handle the extra space requirements-
|
| 498 |
|
|
essentially create an additional patch mirror tier.</p>
|
| 499 |
|
|
<p>2) Mirroring only a patch for certain package versions, rather then full
|
| 500 |
|
|
source. Using kdelibs-3.1.5 as an example, only the patch would be mirrored
|
| 501 |
|
|
(rather then the full 10.53 MB source). Downside to this approach is that a
|
| 502 |
|
|
user who is downloading kdelibs for the first time would either need to pull
|
| 503 |
|
|
it from the original SRC_URI (placing the burden onto the upstream mirror), or
|
| 504 |
|
|
pull the 3.1.4 version, and the patch- pulling 63k more then if they had just
|
| 505 |
|
|
pulled the full version. The kdelibs 3.1.4/3.1.5 example is something of an
|
| 506 |
|
|
optimal case- not all versions will have such miniscule patches.</p>
|
| 507 |
|
|
<p>3) A variation on the idea above, essentially mirroring only the patch for
|
| 508 |
|
|
the oldest version(s) of a package; eg, kdelibs currently has version 3.05,
|
| 509 |
|
|
3.1.5, 3.2.0, and 3.2.1- the mirrors would only carry a patch for 3.05, not
|
| 510 |
|
|
full source (think RESTRICT="fetch"). One plus to this is that patches to
|
| 511 |
|
|
downgrade in version are smaller then the patches to upgrade in version- there
|
| 512 |
|
|
are exceptions to this, but they're hard to find. A major downside to this
|
| 513 |
|
|
approach is A) a user would have to sync up to get the patchlists for that
|
| 514 |
|
|
version, B) creation of a set of patches to go backwards in version (see
|
| 515 |
|
|
<a class="reference" href="#binary-patches-vs-gnudiff-patches">Binary patches vs GNUDiff patches</a>)..</p>
|
| 516 |
|
|
<p>Of the options listed above, the first is the easiest, although the second
|
| 517 |
|
|
could be made to work. Feedback and any possible alternatives would be
|
| 518 |
|
|
greatly appreciated.</p>
|
| 519 |
|
|
</div>
|
| 520 |
g2boojum |
1.4 |
<div class="section">
|
| 521 |
|
|
<h2><a class="toc-backref" href="#id21" id="patch-creation" name="patch-creation">Patch Creation</a></h2>
|
| 522 |
g2boojum |
1.1 |
<p>Maintenance of patch lists, and the actual patch creation ought to be managed
|
| 523 |
|
|
by a high level script- essentally a dev says "I want a patch between this
|
| 524 |
|
|
version, and that version: make it so", the script churns away
|
| 525 |
|
|
creating/updating the patch list, and generating the patch locally. The
|
| 526 |
|
|
utility next uploads the new patch to /space/distfiles-local on dev.gentoo.org
|
| 527 |
|
|
(exempting if it's not a locally generated patch), and repoman is used to
|
| 528 |
|
|
commit the updated patch list.</p>
|
| 529 |
|
|
<p>What would be preferable (although possibly wishful thinking), is if hardware
|
| 530 |
|
|
could be co-opted for automatic patch generation, rather then forcing it upon
|
| 531 |
|
|
the devs- something akin to how files are pulled onto the mirror automatically
|
| 532 |
|
|
for new ebuilds.</p>
|
| 533 |
|
|
<p>The initial bulk of patches to get will be generated by the author, to ease
|
| 534 |
|
|
the transition and offer patches for people to test out.</p>
|
| 535 |
|
|
</div>
|
| 536 |
|
|
</div>
|
| 537 |
g2boojum |
1.4 |
<div class="section">
|
| 538 |
|
|
<h1><a class="toc-backref" href="#id22" id="backwards-compatibility" name="backwards-compatibility">Backwards Compatibility</a></h1>
|
| 539 |
g2boojum |
1.1 |
<p>As noted in <a class="reference" href="#the-proposed-solution">The Proposed Solution</a>, a system using patching and sharing out
|
| 540 |
|
|
it's distfiles must share out it's alternate md5 db. Any system that uses the
|
| 541 |
|
|
distfiles share must support the alternate md5 db also. If this is considered
|
| 542 |
|
|
enough of an issue, it is conceivable to place reconstructed sources with an
|
| 543 |
|
|
alternate md5 into a subdirectory of distdir- portage only looks within
|
| 544 |
|
|
distdir, unwilling to descend into subdirectories.</p>
|
| 545 |
|
|
<p>Also note that <a class="reference" href="#distfile-mirror-additions">Distfile Mirror Additions</a> may add additional backwards
|
| 546 |
g2boojum |
1.3 |
compatibility issues, depending on what solution is accepted.</p>
|
| 547 |
g2boojum |
1.1 |
</div>
|
| 548 |
g2boojum |
1.4 |
<div class="section">
|
| 549 |
|
|
<h1><a class="toc-backref" href="#id23" id="reference-implementation" name="reference-implementation">Reference Implementation</a></h1>
|
| 550 |
g2boojum |
1.1 |
<p>TODO</p>
|
| 551 |
|
|
</div>
|
| 552 |
g2boojum |
1.4 |
<div class="section">
|
| 553 |
|
|
<h1><a class="toc-backref" href="#id24" id="references" name="references">References</a></h1>
|
| 554 |
g2boojum |
1.3 |
<table class="docutils footnote" frame="void" id="id4" rules="none">
|
| 555 |
g2boojum |
1.1 |
<colgroup><col class="label" /><col /></colgroup>
|
| 556 |
|
|
<tbody valign="top">
|
| 557 |
|
|
<tr><td class="label"><a class="fn-backref" href="#id1" name="id4">[1]</a></td><td><a class="reference" href="http://dev.gentoo.org/~ferringb/patches/kdelibs-3.1.4-3.1.5">http://dev.gentoo.org/~ferringb/patches/kdelibs-3.1.4-3.1.5</a>.{patch,diff}.bz2.</td></tr>
|
| 558 |
|
|
</tbody>
|
| 559 |
|
|
</table>
|
| 560 |
g2boojum |
1.3 |
<table class="docutils footnote" frame="void" id="id5" rules="none">
|
| 561 |
g2boojum |
1.1 |
<colgroup><col class="label" /><col /></colgroup>
|
| 562 |
|
|
<tbody valign="top">
|
| 563 |
g2boojum |
1.4 |
<tr><td class="label"><a name="id5">[2]</a></td><td><em>(<a class="fn-backref" href="#id2">1</a>, <a class="fn-backref" href="#id3">2</a>)</em> kdelibs-3.1.4-3.1.5.patch.bz2, switching format patch, created via diffball-0.4_pre4 (diffball is available at <a class="reference" href="http://sourceforge.net/projects/diffball">http://sourceforge.net/projects/diffball</a>)
|
| 564 |
g2boojum |
1.1 |
Bzip2 -9 compressed, the patch is 75,687 bytes, uncompressed it is 337,649 bytes. The patch is available at <a class="reference" href="http://dev.gentoo.org/~ferringb/kdelibs-3.1.4-3.1.5.patch.bz2">http://dev.gentoo.org/~ferringb/kdelibs-3.1.4-3.1.5.patch.bz2</a> for those curious.</td></tr>
|
| 565 |
|
|
</tbody>
|
| 566 |
|
|
</table>
|
| 567 |
g2boojum |
1.3 |
<table class="docutils footnote" frame="void" id="id6" rules="none">
|
| 568 |
g2boojum |
1.1 |
<colgroup><col class="label" /><col /></colgroup>
|
| 569 |
|
|
<tbody valign="top">
|
| 570 |
g2boojum |
1.4 |
<tr><td class="label"><a name="id6">[3]</a></td><td>Glep9, 'Gentoo Package Update System'
|
| 571 |
g2boojum |
1.1 |
(<a class="reference" href="http://glep.gentoo.org/glep-0009.html">http://glep.gentoo.org/glep-0009.html</a>)</td></tr>
|
| 572 |
|
|
</tbody>
|
| 573 |
|
|
</table>
|
| 574 |
|
|
</div>
|
| 575 |
g2boojum |
1.4 |
<div class="section">
|
| 576 |
|
|
<h1><a class="toc-backref" href="#id25" id="copyright" name="copyright">Copyright</a></h1>
|
| 577 |
g2boojum |
1.1 |
<p>This document has been placed in the public domain.</p>
|
| 578 |
|
|
</div>
|
| 579 |
g2boojum |
1.3 |
|
| 580 |
g2boojum |
1.1 |
</div>
|
| 581 |
|
|
<div class="footer">
|
| 582 |
g2boojum |
1.4 |
<hr class="footer" />
|
| 583 |
g2boojum |
1.1 |
<a class="reference" href="glep-0025.txt">View document source</a>.
|
| 584 |
g2boojum |
1.4 |
Generated on: 2006-10-10 20:23 UTC.
|
| 585 |
g2boojum |
1.1 |
Generated by <a class="reference" href="http://docutils.sourceforge.net/">Docutils</a> from <a class="reference" href="http://docutils.sourceforge.net/rst.html">reStructuredText</a> source.
|
| 586 |
g2boojum |
1.4 |
|
| 587 |
g2boojum |
1.1 |
</div>
|
| 588 |
|
|
</body>
|
| 589 |
|
|
</html>
|
| 590 |
|
|
|