W3C home > Mailing lists > Public > www-validator-cvs@w3.org > August 2010

link-checker commit: Add new options:

From: Mercurial notifier <nobody@w3.org>
Date: Thu, 05 Aug 2010 14:47:22 +0000
To: link-checker updates <www-validator-cvs@w3.org>
Message-Id: <E1Oh1j8-0006I6-LY@blinky.w3.org>
changeset:   318:2aa66ff72527
user:        mernst
date:        Mon Jun 29 20:14:38 2009 +0000
files:       bin/checklink bin/checklink.pod
description:
Add new options:
  --suppress-redirect
  --suppress-redirect-prefix
  --suppress-broken
  --suppress-fragment
Also improve initialization code for Exclude_Docs (--exclude-docs option).


diff -r 01b9b5a0b532 -r 2aa66ff72527 bin/checklink
--- a/bin/checklink	Sun May 03 09:36:24 2009 +0000
+++ b/bin/checklink	Mon Jun 29 20:14:38 2009 +0000
@@ -5,7 +5,7 @@
 # (c) 1999-2009 World Wide Web Consortium
 # based on Renaud Bruyeron's checklink.pl
 #
-# $Id: checklink,v 4.160 2009-05-03 09:34:56 ville Exp $
+# $Id: checklink,v 4.161 2009-06-29 20:14:38 mernst Exp $
 #
 # This program is licensed under the W3C(r) Software License:
 #       http://www.w3.org/Consortium/Legal/copyright-software
@@ -275,7 +275,7 @@
   $PROGRAM     = 'W3C-checklink';
   $VERSION     = '4.5';
   $REVISION    = sprintf('version %s (c) 1999-2009 W3C', $VERSION);
-  my ($cvsver) = q$Revision: 4.160 $ =~ /(\d+[\d\.]*\.\d+)/;
+  my ($cvsver) = q$Revision: 4.161 $ =~ /(\d+[\d\.]*\.\d+)/;
   $AGENT       = sprintf('%s/%s [%s] %s',
                          $PROGRAM, $VERSION, $cvsver,
                          (W3C::UserAgent::USE_ROBOT_UA
@@ -392,7 +392,11 @@
     Password          => undef,
     Base_Locations    => [],
     Exclude           => undef,
-    Exclude_Docs      => [],
+    Exclude_Docs      => undef,
+    Suppress_Redirect => [],
+    Suppress_Redirect_Prefix => [],
+    Suppress_Broken   => [],
+    Suppress_Fragment => [],
     Masquerade        => 0,
     Masquerade_From   => '',
     Masquerade_To     => '',
@@ -418,20 +422,6 @@
 my $doc_header;
 
 &parse_arguments() if $Opts{Command_Line};
-
-# Precompile/error-check regular expressions.
-if (defined($Opts{Exclude})) {
-  eval { $Opts{Exclude} = qr/$Opts{Exclude}/o; };
-  &usage(1, "Error in exclude regexp: $@") if $@;
-}
-for my $i (0 .. $#{$Opts{Exclude_Docs}}) {
-  eval { $Opts{Exclude_Docs}->[$i] = qr/$Opts{Exclude_Docs}->[$i]/; };
-  &usage(1, "Error in exclude-docs regexp: $@") if $@;
-}
-if (defined($Opts{Trusted})) {
-  eval { $Opts{Trusted} = qr/$Opts{Trusted}/io; };
-  &usage(1, "Error in trusted domains regexp: $@") if $@;
-}
 
 my $ua = W3C::UserAgent->new($AGENT); # @@@ TODO: admin address
 # @@@ make number of keep-alive connections customizable
@@ -643,6 +633,10 @@
              'l|location=s'    => \@locs,
              'X|exclude=s'     => \$Opts{Exclude},
              'exclude-docs=s@' => \@{$Opts{Exclude_Docs}},
+             'suppress-redirect=s@' => \@{$Opts{Suppress_Redirect}},
+             'suppress-redirect-prefix=s@' => \@{$Opts{Suppress_Redirect_Prefix}},
+             'suppress-broken=s@' => \@{$Opts{Suppress_Broken}},
+             'suppress-fragment=s@' => \@{$Opts{Suppress_Fragment}},
              'u|user=s'        => \$Opts{User},
              'p|password=s'    => \$Opts{Password},
              't|timeout=i'     => \$Opts{Timeout},
@@ -686,6 +680,42 @@
 
   $Opts{Depth} = -1 if ($Opts{Depth} == 0 && @locs);
 
+  # Precompile/error-check regular expressions.
+  if (defined($Opts{Exclude})) {
+    eval { $Opts{Exclude} = qr/$Opts{Exclude}/o; };
+    &usage(1, "Error in exclude regexp: $@") if $@;
+  }
+  for my $i (0 .. $#{$Opts{Exclude_Docs}}) {
+    eval { $Opts{Exclude_Docs}->[$i] = qr/$Opts{Exclude_Docs}->[$i]/; };
+    &usage(1, "Error in exclude-docs regexp: $@") if $@;
+  }
+  if (defined($Opts{Trusted})) {
+    eval { $Opts{Trusted} = qr/$Opts{Trusted}/io; };
+    &usage(1, "Error in trusted domains regexp: $@") if $@;
+  }
+
+  # Sanity-check error-suppression arguments
+  for my $sr_arg (@{$Opts{Suppress_Redirect}}) {
+    if ($sr_arg !~ /.->./) {
+      &usage(1, "Bad suppress-redirect argument, should contain \"->\": $sr_arg");
+    }
+  }
+  for my $srp_arg (@{$Opts{Suppress_Redirect_Prefix}}) {
+    if ($srp_arg !~ /.->./) {
+      &usage(1, "Bad suppress-redirect-prefix argument, should contain \"->\": $srp_arg");
+    }
+  }
+  for my $sb_arg (@{$Opts{Suppress_Broken}}) {
+    if ($sb_arg !~ /^(-1|[0-9]+):./) {
+      &usage(1, "Bad suppress-broken argument: $sb_arg");
+    }
+  }
+  for my $sf_arg (@{$Opts{Suppress_Fragment}}) {
+    if ($sf_arg !~ /.#./) {
+      &usage(1, "Bad suppress-fragment argument, should contain \"#\": $sf_arg");
+    }
+  }
+
   return;
 }
 
@@ -728,6 +758,18 @@
  --exclude-docs REGEXP      In recursive mode, do not check links in documents
                             whose full, canonical URIs match REGEXP.  This
                             option may be specified multiple times.
+ --suppress-redirect URI->URI  Do not report a redirect from the first to the
+                            second URI.  This option may be specified multiple
+                            times.
+ --suppress-redirect-prefix URI->URI  Do not report a redirect from a child of
+                            the first URI to the same child of the second URI.
+                            This option may be specified multiple times.
+ --suppress-broken CODE:URI  Do not report a broken link with the given CODE.
+                            CODE is HTTP response, or -1 for robots exclusion.
+                            This option may be specified multiple times.
+ --suppress-fragment URI    Do not report the given broken fragment URI.
+                            A fragment URI contains \"#\".  This option may be
+                            specified multiple times.
  -L, --languages LANGS      Accept-Language header to send.  The special value
                             'auto' causes autodetection from the environment.
  -R, --no-referer           Do not send the Referer HTTP header.
@@ -1403,6 +1445,19 @@
     $results{$uri}{location}{orig_message} = $tmp->message() || '(no message)';
   }
   $results{$uri}{location}{success} = $response->is_success();
+
+  # If a suppressed broken link, fill the data structure like a typical success.
+  # print STDERR "success? " . $results{$uri}{location}{success} . ": $uri\n";
+  if (! $results{$uri}{location}{success}) {
+    my $code = $results{$uri}{location}{code};
+    my $match = grep { $_ eq "$code:$uri" } @{$Opts{Suppress_Broken}};
+    if ($match) {
+      $results{$uri}{location}{success} = 1;
+      $results{$uri}{location}{code} = 100;
+      $results{$uri}{location}{display} = 100;
+    }
+  }
+
   # Stores the authentication information
   if (defined($response->{Realm})) {
     $results{$uri}{location}{realm} = $response->{Realm};
@@ -1767,7 +1822,8 @@
   # Check that the fragments exist
   foreach my $fragment (keys %{$links->{$uri}{fragments}}) {
     if (defined($p->{Anchors}{$fragment})
-        || &escape_match($fragment, $p->{Anchors})) {
+        || &escape_match($fragment, $p->{Anchors})
+        || grep { $_ eq "$uri#$fragment" } @{$Opts{Suppress_Fragment}}) {
       $results{$uri}{fragments}{$fragment} = 1;
     } else {
       $results{$uri}{fragments}{$fragment} = 0;
@@ -1870,6 +1926,44 @@
 {
   my ($redirects, $response) = @_;
   for (my $prev = $response->previous(); $prev; $prev = $prev->previous()) {
+
+    # Check for redirect match.
+    my $from = $prev->request()->url();
+    my $to = $response->request()->url(); # same on every loop iteration
+    my $from_to = $from . '->' . $to;
+    my $match = grep { $_ eq $from_to } @{$Opts{Suppress_Redirect}};
+    # print STDERR "Result $match of checking $from_to\n";
+    if ($match) { next; }
+
+    # Check for redirect_prefix match
+    my $prefix_match = 0;
+    my $from_len = length($from);
+    my $to_len = length($to);
+    for my $redir_prefix (@{$Opts{Suppresss_Redirect_Prefix}}) {
+      if ($redir_prefix !~ /^(.*)->(.*)$/) {
+        die "Bad suppress-redirect-prefix: $redir_prefix";
+      }
+      my $from_prefix = $1;
+      my $to_prefix = $2;
+      my $from_prefix_len = length($from_prefix);
+      my $to_prefix_len = length($to_prefix);
+      if (($from eq $from_prefix) && ($to eq $to_prefix)) {
+        $prefix_match = 1;
+        last;
+      } elsif (($from_prefix_len < $from_len)
+                 && ($to_prefix_len < $to_len)
+                 && ($from_prefix eq substr($from, 0, $from_prefix_len))
+                 && ($to_prefix eq substr($to, 0, $to_prefix_len))
+                 && (substr($from, $from_prefix_len) eq substr($to, $to_prefix_len))) {
+        $prefix_match = 1;
+        last;
+      }
+    }
+    if ($prefix_match) {
+      # print STDERR "A SUPPRESSED REDIRECT:\n  $from\n  $to\n";
+      next;
+    }
+
     $redirects->{$prev->request()->url()} = $response->request()->url();
   }
   return;
@@ -2119,9 +2213,9 @@
 
              # Response code chain
              join(' <span class="redirected_to" title="redirected to">-&gt;</span> ',
-                  map { &encode($_) } @http_codes),             
+                  map { &encode($_) } @http_codes),
              # HTTP final message
-             $http_message, 
+             $http_message,
              # What to do
              $whattodo,
              # Redirect too?
@@ -2586,8 +2680,8 @@
 </fieldset>
 <p class=\"submit_button\"><input type=\"submit\" name=\"check\" value=\"Check\" /></p>
 </form>
-<div class=\"intro\" id=\"don_program\"></div>  
-<script type=\"text/javascript\" src=\"http://www.w3.org/QA/Tools/don_prog.js\"></script>  
+<div class=\"intro\" id=\"don_program\"></div>
+<script type=\"text/javascript\" src=\"http://www.w3.org/QA/Tools/don_prog.js\"></script>
 ";
   return;
 }
diff -r 01b9b5a0b532 -r 2aa66ff72527 bin/checklink.pod
--- a/bin/checklink.pod	Sun May 03 09:36:24 2009 +0000
+++ b/bin/checklink.pod	Mon Jun 29 20:14:38 2009 +0000
@@ -1,4 +1,4 @@
-$Id: checklink.pod,v 1.25 2009-05-03 09:36:24 ville Exp $
+$Id: checklink.pod,v 1.26 2009-06-29 20:14:38 mernst Exp $
 
 =head1 NAME
 
@@ -78,6 +78,28 @@
 
 In recursive mode, do not check links in documents whose full, canonical
 URIs match I<regexp>.  This option may be specified multiple times.
+
+=item B<--suppress-redirect> I<URI-E<gt>URI>
+
+Do not report a redirect from the first to the second URI.  The "-E<gt>" is
+literal text.  This option may be specified multiple times.
+
+=item B<--suppress-redirect-prefix I<URI-E<gt>URI>
+
+Do not report a redirect from a child of the first URI to the same child of
+the second URI.  The \"->\" is literal text.  This option may be specified
+multiple times.
+
+=item B<--suppress-broken> I<CODE:URI>
+
+Do not report a broken link with the given CODE.  CODE is the HTTP
+response, or -1 for robots exclusion.  The ":" is literal text.  This
+option may be specified multiple times.
+
+=item B<--suppress-fragment> I<URI>
+
+Do not report the given broken fragment URI.  A fragment URI contains "#".
+This option may be specified multiple times.
 
 =item B<-L, --languages> I<accept-language>
 
Received on Thursday, 5 August 2010 14:47:32 UTC

This archive was generated by hypermail 2.3.1 : Wednesday, 7 January 2015 15:17:43 UTC